diff --git a/README.md b/README.md index a4c8b23..b4f90c8 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,7 @@ SimpleSyrup owes a lot to other projects: - [ComfyUI Layer Style Advance](https://github.com/chflame163/ComfyUI_LayerStyle_Advance) provides the SAM model bundle SimpleSyrup can adapt. - [Tiled Diffusion & VAE for AUTOMATIC1111](https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111) informed the practical tiled diffusion and Mixture of Diffusers behavior reimplemented here. - [RES4LYF](https://github.com/ClownsharkBatwing/RES4LYF) is the source of the beta57 scheduler preset reimplemented here. +- [ComfyUI-ppm](https://github.com/pamparamm/ComfyUI-ppm) by pamparamm provides the ModelPatcher-based NegPiP behavior adapted here and builds on the [ComfyUI port](https://github.com/laksjdjf/cd-tuner_negpip-ComfyUI) by laksjdjf and the [original WebUI implementation](https://github.com/hako-mikan/sd-webui-negpip) by hako-mikan. SimpleSyrup also vendors or reimplements selected third-party behavior for SAM-HQ, MobileSAM, GroundingDINO, AUTOMATIC1111 sampler behavior, k-diffusion, and tiled diffusion. See [third_party/NOTICE.md](third_party/NOTICE.md) for the complete notices. diff --git a/simple_syrup/domain/negative_prompt_weights.py b/simple_syrup/domain/negative_prompt_weights.py new file mode 100644 index 0000000..7375821 --- /dev/null +++ b/simple_syrup/domain/negative_prompt_weights.py @@ -0,0 +1,89 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Detect effective negative weights in Comfy-style prompt emphasis.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class _WeightedPromptSegment: + """Retain one parsed prompt fragment and its effective scalar weight.""" + + text: str + weight: float + + +def contains_negative_prompt_weight(text: str) -> bool: + """Return whether valid nested emphasis gives any prompt text a negative weight.""" + + if not isinstance(text, str): + raise TypeError("Negative prompt-weight detection requires text.") + escaped = text.replace(r"\)", "\0\1").replace(r"\(", "\0\2") + return any( + segment.text and segment.weight < 0.0 + for segment in _weighted_segments(escaped, 1.0) + ) + + +def _weighted_segments( + text: str, + current_weight: float, +) -> tuple[_WeightedPromptSegment, ...]: + """Parse emphasis with the same nesting and final-colon rules as ComfyUI.""" + + parsed: list[_WeightedPromptSegment] = [] + for item in _parenthesized_items(text): + weight = current_weight + if len(item) >= 2 and item[0] == "(" and item[-1] == ")": + inner = item[1:-1] + delimiter = inner.rfind(":") + weight *= 1.1 + if delimiter > 0: + try: + weight = float(inner[delimiter + 1 :]) + except ValueError: + pass + else: + inner = inner[:delimiter] + parsed.extend(_weighted_segments(inner, weight)) + continue + parsed.append( + _WeightedPromptSegment( + item.replace("\0\1", ")").replace("\0\2", "("), + current_weight, + ) + ) + return tuple(parsed) + + +def _parenthesized_items(text: str) -> tuple[str, ...]: + """Split top-level parenthesized regions while preserving malformed input.""" + + result: list[str] = [] + current = "" + nesting = 0 + for character in text: + if character == "(": + if nesting == 0: + if current: + result.append(current) + current = "(" + else: + current += character + nesting += 1 + elif character == ")": + nesting -= 1 + if nesting == 0: + result.append(f"{current})") + current = "" + else: + current += character + else: + current += character + if current: + result.append(current) + return tuple(result) diff --git a/simple_syrup/nodes_v3/__init__.py b/simple_syrup/nodes_v3/__init__.py index 341c303..99af78a 100644 --- a/simple_syrup/nodes_v3/__init__.py +++ b/simple_syrup/nodes_v3/__init__.py @@ -135,6 +135,7 @@ def get_nodes() -> list[type[object]]: if not prompt_control_is_available(): return nodes + from .apply_automatic_negpip import ApplyAutomaticNegpipV3 from .attach_regional_global_conditioning import ( AttachRegionalGlobalConditioningV3, ) @@ -149,6 +150,7 @@ def get_nodes() -> list[type[object]]: return [ *nodes, + ApplyAutomaticNegpipV3, AttachRegionalGlobalConditioningV3, EncodePromptBatchWithPromptControl, LabelRegionalLoraHooksV3, diff --git a/simple_syrup/nodes_v3/apply_automatic_negpip.py b/simple_syrup/nodes_v3/apply_automatic_negpip.py new file mode 100644 index 0000000..ab88316 --- /dev/null +++ b/simple_syrup/nodes_v3/apply_automatic_negpip.py @@ -0,0 +1,69 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Internal Comfy v3 node for model-family automatic NegPiP preparation.""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING, Any + +from ..services.negpip_model_service import NEGPIP_MODEL_SERVICE + +if TYPE_CHECKING: + + class _ComfyNodeBase: + """Type-checking base for Comfy v3 nodes.""" + + pass + +else: + _ComfyNodeBase = import_module("comfy_api.latest").io.ComfyNode + +_comfy_io: Any = None if TYPE_CHECKING else import_module("comfy_api.latest").io + + +class ApplyAutomaticNegpipV3(_ComfyNodeBase): + """Patch supported MODEL/CLIP pairs after a negative prompt-weight trigger.""" + + @classmethod + def define_schema(cls) -> Any: + """Declare the internal runtime patch boundary.""" + + return _comfy_io.Schema( + node_id="SimpleSyrup.ApplyAutomaticNegpip", + display_name="Apply Automatic NegPiP (Internal)", + category="SimpleSyrup/Internal", + description=( + "Internal model-family NegPiP preparation injected by Schedule & " + "Encode Prompts after detecting a negative prompt weight." + ), + is_dev_only=True, + inputs=[ + _comfy_io.Model.Input( + "model", + tooltip="MODEL inspected and cloned only when NegPiP is supported.", + ), + _comfy_io.Clip.Input( + "clip", + tooltip="CLIP cloned with the matching NegPiP encoder behavior.", + ), + ], + outputs=[ + _comfy_io.Model.Output( + "model", + tooltip="MODEL carrying one supported NegPiP attention patch set.", + ), + _comfy_io.Clip.Output( + "clip", + tooltip="CLIP carrying matching negative-weight encoding behavior.", + ), + ], + ) + + @classmethod + def execute(cls, model: object, clip: object) -> tuple[object, object]: + """Return the supported patched pair or the original unsupported pair.""" + + return NEGPIP_MODEL_SERVICE.prepare(model, clip) diff --git a/simple_syrup/runtime/clip_patcher_mutations.py b/simple_syrup/runtime/clip_patcher_mutations.py index 89f6c3c..4a6be5e 100644 --- a/simple_syrup/runtime/clip_patcher_mutations.py +++ b/simple_syrup/runtime/clip_patcher_mutations.py @@ -6,6 +6,7 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from typing import Any, cast @@ -50,6 +51,79 @@ class ClipHookScheduleMutation: register_hooks(self.hooks, self.target) +@dataclass(frozen=True) +class ClipCallableObjectPatchMutation: + """Patch one callable text-encoder object on a derived CLIP patcher.""" + + path: str + replacement: Callable[..., object] + + def apply(self, clip: object) -> None: + """Validate the path and collision state before installing the callback.""" + + if ( + not isinstance(self.path, str) + or not self.path + or any(not segment for segment in self.path.split(".")) + ): + raise ValueError("CLIP callable patch path must be a dotted path.") + if not callable(self.replacement): + raise TypeError("CLIP callable object replacement must be callable.") + patcher = _required_attribute(clip, "patcher", value_name="CLIP") + getter = getattr(patcher, "get_model_object", None) + adder = getattr(patcher, "add_object_patch", None) + object_patches = getattr(patcher, "object_patches", None) + if ( + not callable(getter) + or not callable(adder) + or not isinstance(object_patches, dict) + ): + raise TypeError("CLIP patcher does not expose callable object patches.") + if self.path in object_patches: + raise ValueError(f"CLIP object path '{self.path}' already has a patch.") + if not callable(getter(self.path)): + raise TypeError(f"CLIP object path '{self.path}' must be callable.") + adder(self.path, self.replacement) + + +@dataclass(frozen=True) +class ClipTokenizerMutation: + """Replace the tokenizer on a derived CLIP after exact source validation.""" + + expected_source: object + replacement: object + + def apply(self, clip: object) -> None: + """Install one tokenizer proxy only on the expected cloned source value.""" + + if getattr(clip, "tokenizer", None) is not self.expected_source: + raise ValueError("Derived CLIP tokenizer does not match its source.") + cast(Any, clip).tokenizer = self.replacement + + +@dataclass(frozen=True) +class ClipBooleanOptionMutation: + """Publish one collision-safe boolean option on a derived CLIP patcher.""" + + key: str + value: bool + + def apply(self, clip: object) -> None: + """Set an approved ownership marker after validating the option mapping.""" + + if self.key not in {"ppm_negpip", "simple_syrup_negpip"}: + raise ValueError("Unsupported CLIP boolean option marker.") + if not isinstance(self.value, bool): + raise TypeError("CLIP option marker value must be boolean.") + patcher = _required_attribute(clip, "patcher", value_name="CLIP") + options = getattr(patcher, "model_options", None) + if not isinstance(options, dict): + raise TypeError("CLIP patcher model_options must be a dictionary.") + if self.key in options: + raise ValueError(f"CLIP option '{self.key}' is already present.") + options[self.key] = self.value + + def _required_attribute(value: object, name: str, *, value_name: str) -> object: """Return a required dynamic ComfyUI boundary attribute.""" diff --git a/simple_syrup/runtime/model_patcher_mutations.py b/simple_syrup/runtime/model_patcher_mutations.py index fc864f2..ace9e45 100644 --- a/simple_syrup/runtime/model_patcher_mutations.py +++ b/simple_syrup/runtime/model_patcher_mutations.py @@ -237,6 +237,102 @@ class ModelDiffusionWrapperMutation: ).apply(model) +@dataclass(frozen=True) +class ModelInteropDiffusionWrapperMutation: + """Install the exact legacy key required for PPM Anima interoperability.""" + + key: str + wrapper: Callable[..., object] + + def apply(self, model: object) -> None: + """Install only the documented PPM Anima wrapper surface.""" + + if self.key != "ppm_negpip_anima": + raise ValueError("NegPiP interop wrapper must use PPM's Anima key.") + getter = _require_bound_method(model, "get_wrappers", ("wrapper_type", "key")) + adder = _require_bound_method( + model, + "add_wrapper_with_key", + ("wrapper_type", "key", "wrapper"), + ) + existing = getter(WrappersMP.DIFFUSION_MODEL, self.key) + if not isinstance(existing, list) or any( + not callable(callback) for callback in existing + ): + raise TypeError("Existing NegPiP wrappers must be a callable list.") + if existing: + raise ValueError("PPM's Anima NegPiP wrapper key is already installed.") + adder(WrappersMP.DIFFUSION_MODEL, self.key, self.wrapper) + + +@dataclass(frozen=True) +class ModelAttentionPatchMutation: + """Append one validated Comfy attention patch to a derived MODEL.""" + + patch_name: str + callback: Callable[..., object] + + def apply(self, model: object) -> None: + """Install an attn1 or attn2 callback through the public patcher setter.""" + + if self.patch_name not in {"attn1", "attn2"}: + raise ValueError("MODEL attention patch name must be 'attn1' or 'attn2'.") + if not callable(self.callback): + raise TypeError("MODEL attention patch callback must be callable.") + setter = getattr(model, f"set_model_{self.patch_name}_patch", None) + if not callable(setter): + raise TypeError(f"MODEL does not support {self.patch_name} patches.") + setter(self.callback) + + +@dataclass(frozen=True) +class ModelBooleanOptionMutation: + """Publish one collision-safe boolean MODEL option marker.""" + + key: str + value: bool + + def apply(self, model: object) -> None: + """Set one supported marker only when no value already owns the key.""" + + if self.key != "ppm_negpip": + raise ValueError("Unsupported MODEL boolean option marker.") + if not isinstance(self.value, bool): + raise TypeError("MODEL option marker value must be boolean.") + options = _require_dictionary_attribute(model, "model_options") + if self.key in options: + raise ValueError(f"MODEL option '{self.key}' is already present.") + options[self.key] = self.value + + +@dataclass(frozen=True) +class ModelCallableObjectPatchMutation: + """Replace one callable model object after collision validation.""" + + path: str + replacement: Callable[..., object] + + def apply(self, model: object) -> None: + """Patch one callable path without relying on bound-method identity.""" + + if ( + not isinstance(self.path, str) + or not self.path + or any(not segment for segment in self.path.split(".")) + ): + raise ValueError("MODEL callable patch path must be a dotted path.") + if not callable(self.replacement): + raise TypeError("MODEL callable object replacement must be callable.") + getter = _require_bound_method(model, "get_model_object", ("name",)) + adder = _require_bound_method(model, "add_object_patch", ("name", "obj")) + object_patches = _require_dictionary_attribute(model, "object_patches") + if self.path in object_patches: + raise ValueError(f"MODEL object path '{self.path}' already has a patch.") + if not callable(getter(self.path)): + raise TypeError(f"MODEL object path '{self.path}' must be callable.") + adder(self.path, self.replacement) + + @dataclass(frozen=True) class ModelExactObjectPatchMutation: """Replace one exact model object after collision and identity validation.""" diff --git a/simple_syrup/runtime/negpip/__init__.py b/simple_syrup/runtime/negpip/__init__.py new file mode 100644 index 0000000..e074f0a --- /dev/null +++ b/simple_syrup/runtime/negpip/__init__.py @@ -0,0 +1,5 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Provide model-family NegPiP runtime adapters.""" diff --git a/simple_syrup/runtime/negpip/anima.py b/simple_syrup/runtime/negpip/anima.py new file mode 100644 index 0000000..6b641e9 --- /dev/null +++ b/simple_syrup/runtime/negpip/anima.py @@ -0,0 +1,108 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Apply PPM-compatible value-mask NegPiP behavior to Anima.""" + +# NegPiP behavior is adapted from ComfyUI-ppm and its credited predecessors. +# See third_party/manifest.toml and third_party/NOTICE.md. + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import torch +from comfy import conds + +WRAPPER_KEY = "ppm_negpip_anima" +CONDITION_MASK_KEY = "c_ppm_negpip_mask" +TRANSFORMER_MASK_KEY = "ppm_negpip_mask" + + +def anima_extra_conds_negpip_wrapper( + previous_extra_conds: Callable[..., dict[str, object]], +) -> Callable[..., dict[str, object]]: + """Convert signed T5 weights into a model condition while preserving magnitude.""" + + def wrapped_extra_conds(**kwargs: object) -> dict[str, object]: + """Publish a sequence-aligned value multiplier for one conditioning.""" + + weights = kwargs.get("t5xxl_weights") + multiplier: torch.Tensor | None = None + if weights is not None: + if not isinstance(weights, torch.Tensor): + raise TypeError("Anima NegPiP T5 weights must be a tensor.") + magnitude = weights.abs() + multiplier = ( + torch.where( + weights < 0.0, + weights.new_tensor(-1.0), + weights.new_tensor(1.0), + ) + .unsqueeze(0) + .unsqueeze(-1) + ) + if multiplier.shape[1] < 512: + multiplier = torch.nn.functional.pad( + multiplier, + (0, 0, 0, 512 - multiplier.shape[1]), + value=1.0, + ) + kwargs["t5xxl_weights"] = magnitude + + output = previous_extra_conds(**kwargs) + if not isinstance(output, dict): + raise TypeError("Anima extra conditions must be a dictionary.") + if multiplier is not None: + output[CONDITION_MASK_KEY] = conds.CONDRegular(multiplier) + return output + + return wrapped_extra_conds + + +def anima_diffusion_negpip_wrapper( + executor: Callable[..., object], + *args: object, + **kwargs: object, +) -> object: + """Move the processed Anima multiplier into isolated transformer options.""" + + if len(args) < 3 or not isinstance(args[2], torch.Tensor): + raise TypeError("Anima NegPiP wrapper requires tensor conditioning context.") + context = args[2] + transformer_options = kwargs.get("transformer_options", {}) + if not isinstance(transformer_options, dict): + raise TypeError("Anima transformer options must be a dictionary.") + prepared = transformer_options.copy() + multiplier = kwargs.get(CONDITION_MASK_KEY) + if multiplier is not None: + if not isinstance(multiplier, torch.Tensor): + raise TypeError("Anima NegPiP multiplier must be a tensor.") + prepared[TRANSFORMER_MASK_KEY] = multiplier.to(context) + kwargs["transformer_options"] = prepared + return executor(*args, **kwargs) + + +def anima_attn2_negpip( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + pe: torch.Tensor | None = None, + attn_mask: torch.Tensor | None = None, + extra_options: dict[str, Any] | None = None, +) -> dict[str, torch.Tensor | None]: + """Apply the signed multiplier only to Anima cross-attention values.""" + + multiplier = ( + None if extra_options is None else extra_options.get(TRANSFORMER_MASK_KEY) + ) + if multiplier is not None and not isinstance(multiplier, torch.Tensor): + raise TypeError("Anima NegPiP attention multiplier must be a tensor.") + return { + "q": query, + "k": key, + "v": value if multiplier is None else value * multiplier, + "pe": pe, + "attn_mask": attn_mask, + } diff --git a/simple_syrup/runtime/negpip/krea2.py b/simple_syrup/runtime/negpip/krea2.py new file mode 100644 index 0000000..51896e3 --- /dev/null +++ b/simple_syrup/runtime/negpip/krea2.py @@ -0,0 +1,291 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Adapt NegPiP value masking to Krea 2's layered Qwen conditioning.""" + +# NegPiP behavior is adapted from ComfyUI-ppm and its credited predecessors. +# See third_party/manifest.toml and third_party/NOTICE.md. + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import Any + +import torch +from comfy import conds + +CLIP_MARKER = "simple_syrup_negpip" +WRAPPER_KEY = "simple_syrup.negpip.krea2" +ENCODER_MASK_KEY = "simple_syrup_negpip_mask" +CONDITION_MASK_KEY = "c_simple_syrup_negpip_mask" +TRANSFORMER_MASK_KEY = "simple_syrup_negpip_mask" +KREA_TOKEN_KEY = "qwen3vl_4b" +IM_START_TOKEN = 151644 +USER_TOKEN = 872 +NEWLINE_TOKEN = 198 +IMAGE_PAD_TOKEN = 151655 + + +class Krea2NegpipTokenizer: + """Preserve Krea templates while enabling Comfy prompt-weight tokenization.""" + + def __init__(self, source: object) -> None: + """Retain one cloned CLIP's shared source tokenizer without mutating it.""" + + self._source = source + + def __getattr__(self, name: str) -> object: + """Delegate tokenizer metadata and helpers to the installed Krea tokenizer.""" + + return getattr(self._source, name) + + def tokenize_with_weights( + self, + text: str, + return_word_ids: bool = False, + llama_template: str | None = None, + images: Sequence[torch.Tensor] = (), + prevent_empty_text: bool = False, + thinking: bool = True, + **kwargs: object, + ) -> dict[str, list[list[tuple[object, ...]]]]: + """Tokenize the normal Krea template while retaining parsed scalar weights.""" + + image = kwargs.pop("image", None) + if image is not None and not images: + if not isinstance(image, torch.Tensor): + raise TypeError("Krea tokenizer image input must be a tensor.") + images = tuple(image[index : index + 1] for index in range(image.shape[0])) + skip_template = bool(kwargs.pop("skip_template", False)) or text.startswith( + "<|im_start|>" + ) + kwargs.pop("disable_weights", None) + if prevent_empty_text and text == "": + text = " " + + if skip_template: + prepared_text = text + else: + template = llama_template + if template is None: + template_name = ( + "llama_template" if not images else "llama_template_images" + ) + template = getattr(self._source, template_name) + if not isinstance(template, str): + raise TypeError("Krea tokenizer template must be text.") + if len(images) > 1: + vision_block = "<|vision_start|><|image_pad|><|vision_end|>" + template = template.replace( + vision_block, + vision_block * len(images), + 1, + ) + prepared_text = template.format(text) + if not thinking: + prepared_text += "\n\n\n\n" + + inner = getattr(self._source, KREA_TOKEN_KEY) + tokens = inner.tokenize_with_weights( + prepared_text, + return_word_ids=return_word_ids, + disable_weights=False, + **kwargs, + ) + embedded_count = 0 + for section in tokens: + for index, pair in enumerate(section): + token = pair[0] + if ( + isinstance(token, (int, float)) + and token == IMAGE_PAD_TOKEN + and embedded_count < len(images) + ): + section[index] = ( + { + "type": "image", + "data": images[embedded_count], + "original_type": "image", + }, + *pair[1:], + ) + embedded_count += 1 + return {KREA_TOKEN_KEY: tokens} + + +def encode_krea2_token_weights_negpip( + original: Callable[..., tuple[object, ...]], + token_weight_pairs: dict[str, list[list[tuple[object, ...]]]], + template_end: int = -1, +) -> tuple[object, ...]: + """Encode absolute Krea magnitudes and publish a post-template sign mask.""" + + sections = token_weight_pairs.get(KREA_TOKEN_KEY) + if not isinstance(sections, list) or len(sections) != 1: + raise ValueError("Krea NegPiP requires exactly one Qwen token section.") + source_section = sections[0] + absolute_section = [ + (pair[0], abs(_token_weight(pair)), *pair[2:]) for pair in source_section + ] + absolute_tokens = dict(token_weight_pairs) + absolute_tokens[KREA_TOKEN_KEY] = [absolute_section] + encoded = original(absolute_tokens, template_end=template_end) + if len(encoded) < 3 or not isinstance(encoded[0], torch.Tensor): + raise TypeError("Krea NegPiP encoder must return tensor conditioning metadata.") + extra = encoded[2] + if not isinstance(extra, dict): + raise TypeError("Krea NegPiP encoder metadata must be a dictionary.") + cut = _template_end(source_section) if template_end == -1 else template_end + signs = [ + -1.0 if _token_weight(pair) < 0.0 else 1.0 for pair in source_section[cut:] + ] + sequence_length = int(encoded[0].shape[1]) + if len(signs) != sequence_length: + raise ValueError( + "Krea NegPiP sign mask does not match post-template conditioning: " + f"{len(signs)} signs for {sequence_length} tokens." + ) + prepared_extra = dict(extra) + prepared_extra[ENCODER_MASK_KEY] = torch.tensor(signs).reshape(1, -1, 1) + return encoded[0], encoded[1], prepared_extra + + +def krea2_extra_conds_negpip_wrapper( + previous_extra_conds: Callable[..., dict[str, object]], +) -> Callable[..., dict[str, object]]: + """Publish the Krea token-sign mask as a processed model condition.""" + + def wrapped_extra_conds(**kwargs: object) -> dict[str, object]: + """Attach a validated sequence multiplier without altering other conditions.""" + + output = previous_extra_conds(**kwargs) + if not isinstance(output, dict): + raise TypeError("Krea extra conditions must be a dictionary.") + multiplier = kwargs.get(ENCODER_MASK_KEY) + if multiplier is not None: + if not isinstance(multiplier, torch.Tensor): + raise TypeError("Krea NegPiP sign mask must be a tensor.") + if ( + multiplier.ndim != 3 + or multiplier.shape[0] != 1 + or multiplier.shape[2] != 1 + ): + raise ValueError( + "Krea NegPiP sign mask must have shape (1, sequence, 1)." + ) + output[CONDITION_MASK_KEY] = conds.CONDRegular(multiplier) + return output + + return wrapped_extra_conds + + +def krea2_diffusion_negpip_wrapper( + executor: Callable[..., object], + *args: object, + **kwargs: object, +) -> object: + """Move a processed Krea sign mask into call-local transformer options.""" + + positional_options = args[5] if len(args) > 5 else None + transformer_options = ( + positional_options + if positional_options is not None + else kwargs.get("transformer_options", {}) + ) + if not isinstance(transformer_options, dict): + raise TypeError("Krea transformer options must be a dictionary.") + prepared = transformer_options.copy() + multiplier = kwargs.get(CONDITION_MASK_KEY) + if multiplier is not None: + if not isinstance(multiplier, torch.Tensor): + raise TypeError("Krea NegPiP processed mask must be a tensor.") + prepared[TRANSFORMER_MASK_KEY] = multiplier + if len(args) > 5: + prepared_args = list(args) + prepared_args[5] = prepared + return executor(*prepared_args, **kwargs) + kwargs["transformer_options"] = prepared + return executor(*args, **kwargs) + + +def krea2_attn1_negpip( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + pe: torch.Tensor | None = None, + attn_mask: torch.Tensor | None = None, + extra_options: dict[str, Any] | None = None, +) -> dict[str, torch.Tensor | None]: + """Apply negative signs only to Krea text values in the joint token stream.""" + + options = {} if extra_options is None else extra_options + multiplier = options.get(TRANSFORMER_MASK_KEY) + if multiplier is None: + return {"q": query, "k": key, "v": value, "pe": pe, "attn_mask": attn_mask} + if not isinstance(multiplier, torch.Tensor): + raise TypeError("Krea NegPiP attention mask must be a tensor.") + image_slice = options.get("img_slice") + if ( + not isinstance(image_slice, (list, tuple)) + or len(image_slice) != 2 + or any( + isinstance(item, bool) or not isinstance(item, int) for item in image_slice + ) + ): + raise ValueError("Krea NegPiP requires the model's text/image token boundary.") + text_length = image_slice[0] + if text_length != multiplier.shape[1] or value.shape[2] < text_length: + raise ValueError( + "Krea NegPiP mask does not match the joint attention sequence." + ) + if multiplier.shape[0] not in {1, value.shape[0]} or multiplier.shape[2] != 1: + raise ValueError("Krea NegPiP mask has an incompatible batch or channel shape.") + text_multiplier = multiplier.to(device=value.device, dtype=value.dtype).unsqueeze(1) + prepared_value = value.to(copy=True) + prepared_value[:, :, :text_length, :] *= text_multiplier + return { + "q": query, + "k": key, + "v": prepared_value, + "pe": pe, + "attn_mask": attn_mask, + } + + +def _token_weight(pair: tuple[object, ...]) -> float: + """Return one finite scalar token weight from a tokenizer tuple.""" + + if ( + len(pair) < 2 + or isinstance(pair[1], bool) + or not isinstance(pair[1], (int, float)) + ): + raise TypeError("Krea token weights must be numeric.") + weight = float(pair[1]) + if not torch.isfinite(torch.tensor(weight)): + raise ValueError("Krea token weights must be finite.") + return weight + + +def _template_end(section: list[tuple[object, ...]]) -> int: + """Resolve the exact Krea system and user-opening prefix boundary.""" + + count = 0 + template_end = -1 + for index, pair in enumerate(section): + token = pair[0] + if ( + not isinstance(token, torch.Tensor) + and token == IM_START_TOKEN + and count < 2 + ): + template_end = index + count += 1 + if ( + len(section) > template_end + 3 + and section[template_end + 1][0] == USER_TOKEN + and section[template_end + 2][0] == NEWLINE_TOKEN + ): + template_end += 3 + return template_end diff --git a/simple_syrup/runtime/negpip/standard.py b/simple_syrup/runtime/negpip/standard.py new file mode 100644 index 0000000..2e29b79 --- /dev/null +++ b/simple_syrup/runtime/negpip/standard.py @@ -0,0 +1,123 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Apply PPM-compatible NegPiP encoding for standard cross-attention models.""" + +# NegPiP behavior is adapted from ComfyUI-ppm and its credited predecessors. +# See third_party/manifest.toml and third_party/NOTICE.md. + +from __future__ import annotations + +from typing import Any + +import torch +from comfy import model_management +from comfy.sd1_clip import SDClipModel, gen_empty_tokens + + +def standard_attn2_negpip( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + extra_options: dict[str, Any], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Select magnitude embeddings for keys and signed embeddings for values.""" + + del extra_options + return query, key[:, 0::2], value[:, 1::2] + + +def encode_token_weights_negpip( + encoder: SDClipModel, + token_weight_pairs: list[list[tuple[object, float]]], +) -> tuple[object, ...]: + """Encode absolute prompt magnitude and interleave signed value embeddings.""" + + tokens_to_encode: list[list[object]] = [] + maximum_length = 0 + has_weights = False + for section in token_weight_pairs: + tokens = [pair[0] for pair in section] + maximum_length = max(len(tokens), maximum_length) + has_weights = has_weights or any(pair[1] != 1.0 for pair in section) + tokens_to_encode.append(tokens) + + section_count = len(tokens_to_encode) + if has_weights or section_count == 0: + if hasattr(encoder, "gen_empty_tokens"): + empty_tokens = encoder.gen_empty_tokens( + encoder.special_tokens, + maximum_length, + ) + else: + empty_tokens = gen_empty_tokens(encoder.special_tokens, maximum_length) + tokens_to_encode.append(empty_tokens) + + encoded = encoder.encode(tokens_to_encode) + output_tensor, pooled = encoded[:2] + if not isinstance(output_tensor, torch.Tensor): + raise TypeError("NegPiP text encoder output must be a tensor.") + first_pooled = ( + pooled[0:1].to(device=model_management.intermediate_device()) + if isinstance(pooled, torch.Tensor) + else pooled + ) + + outputs: list[torch.Tensor] = [] + for section_index in range(section_count): + key_embedding = output_tensor[section_index : section_index + 1].to(copy=True) + value_embedding = key_embedding.to(copy=True) + if has_weights: + empty_embedding = output_tensor[-1] + for batch_index in range(len(key_embedding)): + for token_index in range(len(key_embedding[batch_index])): + weight = token_weight_pairs[section_index][token_index][1] + if weight == 1.0: + continue + magnitude = abs(weight) + key_embedding[batch_index][token_index] = ( + key_embedding[batch_index][token_index] + - empty_embedding[token_index] + ) * magnitude + empty_embedding[token_index] + value_embedding[batch_index][token_index] = ( + value_embedding[batch_index][token_index] + - empty_embedding[token_index] + ) * magnitude + empty_embedding[token_index] + if weight < 0.0: + value_embedding[batch_index][token_index].neg_() + + interleaved = torch.zeros_like(key_embedding).repeat(1, 2, 1) + interleaved[:, 0::2, :] = key_embedding + interleaved[:, 1::2, :] = value_embedding + outputs.append(interleaved) + + if outputs: + result: tuple[object, ...] = ( + torch.cat(outputs, dim=-2).to( + device=model_management.intermediate_device() + ), + first_pooled, + ) + else: + result = ( + output_tensor[-1:].to(device=model_management.intermediate_device()), + first_pooled, + ) + + if len(encoded) <= 2: + return result + source_extra = encoded[2] + if not isinstance(source_extra, dict): + raise TypeError("NegPiP text encoder metadata must be a dictionary.") + extra: dict[str, object] = {} + for key, value in source_extra.items(): + if key == "attention_mask" and isinstance(value, torch.Tensor): + value = ( + value[:section_count] + .flatten() + .unsqueeze(dim=0) + .to(device=model_management.intermediate_device()) + ) + extra[str(key)] = value + return (*result, extra) diff --git a/simple_syrup/runtime/ppm_negpip_interop.py b/simple_syrup/runtime/ppm_negpip_interop.py index b162436..7847101 100644 --- a/simple_syrup/runtime/ppm_negpip_interop.py +++ b/simple_syrup/runtime/ppm_negpip_interop.py @@ -22,21 +22,30 @@ _ANIMA_CONDITION_KEY = "c_ppm_negpip_mask" _ANIMA_TRANSFORMER_KEY = "ppm_negpip_mask" _EXTRA_CONDS_PATH = "extra_conds" _ATTN2_PATCH_NAME = "attn2_patch" -_UNET_CALLBACK = ( - "src.negpip.unet_negpip", - "sdxl_attn2_negpip", +_UNET_CALLBACKS = ( + ("src.negpip.unet_negpip", "sdxl_attn2_negpip"), + ("simple_syrup.runtime.negpip.standard", "standard_attn2_negpip"), ) -_ANIMA_CALLBACK = ( - "src.negpip.anima_negpip", - "cosmos_attn2_negpip", +_ANIMA_CALLBACKS = ( + ("src.negpip.anima_negpip", "cosmos_attn2_negpip"), + ("simple_syrup.runtime.negpip.anima", "anima_attn2_negpip"), ) -_ANIMA_WRAPPER = ( - "src.negpip.anima_negpip", - "cosmos_diffusion_negpip_wrapper", +_ANIMA_WRAPPERS = ( + ("src.negpip.anima_negpip", "cosmos_diffusion_negpip_wrapper"), + ( + "simple_syrup.runtime.negpip.anima", + "anima_diffusion_negpip_wrapper", + ), ) -_ANIMA_EXTRA_CONDS = ( - "src.negpip.anima_negpip", - "anima_extra_conds_negpip_wrapper.._anima_extra_conds_negpip_wrapper", +_ANIMA_EXTRA_CONDS_CALLBACKS = ( + ( + "src.negpip.anima_negpip", + "anima_extra_conds_negpip_wrapper.._anima_extra_conds_negpip_wrapper", + ), + ( + "simple_syrup.runtime.negpip.anima", + "anima_extra_conds_negpip_wrapper..wrapped_extra_conds", + ), ) @@ -135,10 +144,12 @@ class PpmNegpipInteropValidator: extra_conds = object_patches.get(_EXTRA_CONDS_PATH) recognized_surface = any( ( - any(_is_identity(item, *_UNET_CALLBACK) for item in attention), - any(_is_identity(item, *_ANIMA_CALLBACK) for item in attention), + any(_matches_any_identity(item, _UNET_CALLBACKS) for item in attention), + any( + _matches_any_identity(item, _ANIMA_CALLBACKS) for item in attention + ), bool(anima_wrappers), - _is_identity(extra_conds, *_ANIMA_EXTRA_CONDS), + _matches_any_identity(extra_conds, _ANIMA_EXTRA_CONDS_CALLBACKS), ) ) if not marker: @@ -173,9 +184,9 @@ class PpmNegpipInteropValidator: if ( len(attention) != 1 - or not _is_identity(attention[0], *_UNET_CALLBACK) + or not _matches_any_identity(attention[0], _UNET_CALLBACKS) or anima_wrappers - or _is_identity(extra_conds, *_ANIMA_EXTRA_CONDS) + or _matches_any_identity(extra_conds, _ANIMA_EXTRA_CONDS_CALLBACKS) ): raise ValueError( "Standard UNet NegPiP requires exactly its PPM split-K/V attention " @@ -197,10 +208,10 @@ class PpmNegpipInteropValidator: if ( len(attention) != 1 - or not _is_identity(attention[0], *_ANIMA_CALLBACK) + or not _matches_any_identity(attention[0], _ANIMA_CALLBACKS) or len(anima_wrappers) != 1 - or not _is_identity(anima_wrappers[0], *_ANIMA_WRAPPER) - or not _is_identity(extra_conds, *_ANIMA_EXTRA_CONDS) + or not _matches_any_identity(anima_wrappers[0], _ANIMA_WRAPPERS) + or not _matches_any_identity(extra_conds, _ANIMA_EXTRA_CONDS_CALLBACKS) ): raise ValueError( "Anima NegPiP requires exactly its PPM attention patch, keyed " @@ -230,4 +241,13 @@ def _is_identity( ) +def _matches_any_identity( + value: object, + identities: tuple[tuple[str, str], ...], +) -> bool: + """Match a callable against either the installed PPM or owned equivalent.""" + + return any(_is_identity(value, *identity) for identity in identities) + + PPM_NEGPIP_INTEROP_VALIDATOR = PpmNegpipInteropValidator() diff --git a/simple_syrup/runtime/prompt_control_graph_adapter.py b/simple_syrup/runtime/prompt_control_graph_adapter.py index 62a805a..75e0ba0 100644 --- a/simple_syrup/runtime/prompt_control_graph_adapter.py +++ b/simple_syrup/runtime/prompt_control_graph_adapter.py @@ -86,6 +86,28 @@ class PromptControlGraphAdapter: self.merge_expand(expand, negative.expand, "negative LoRA scheduling") return negative.args[0], negative.args[1] + def apply_automatic_negpip( + self, + *, + model: Any, + clip: Any, + expand: dict[str, dict[str, Any]], + ) -> tuple[Any, Any]: + """Insert the runtime family check after a negative prompt-weight trigger.""" + + graph = self._graph_utils.GraphBuilder() + prepared = graph.node( + "SimpleSyrup.ApplyAutomaticNegpip", + model=model, + clip=clip, + ) + self.merge_expand( + expand, + cast(dict[str, dict[str, Any]], graph.finalize()), + "automatic NegPiP preparation", + ) + return prepared.out(0), prepared.out(1) + def encode_segment( self, *, diff --git a/simple_syrup/runtime/prompt_control_schedule_encode_graph.py b/simple_syrup/runtime/prompt_control_schedule_encode_graph.py index 43dbb76..0a8cd71 100644 --- a/simple_syrup/runtime/prompt_control_schedule_encode_graph.py +++ b/simple_syrup/runtime/prompt_control_schedule_encode_graph.py @@ -8,6 +8,7 @@ from __future__ import annotations from typing import Any +from ..domain.negative_prompt_weights import contains_negative_prompt_weight from ..domain.prompt_batch_parser import DEFAULT_PROMPT_BATCH_SEPARATOR from ..domain.prompt_control_prompt import PreparedPromptSide, apply_encode_style from ..services.prompt_control_segment_planning_service import ( @@ -48,6 +49,12 @@ class PromptControlScheduleEncodeGraphBuilder: ) adapter = self.graph_adapter_class.load(PROMPT_CONTROL_MISSING_MESSAGE) expand: dict[str, dict[str, Any]] = {} + if self._requires_negpip(plan): + model, clip = adapter.apply_automatic_negpip( + model=model, + clip=clip, + expand=expand, + ) scheduled_model, encoding_clip = self._sampling_inputs( model=model, clip=clip, @@ -84,6 +91,16 @@ class PromptControlScheduleEncodeGraphBuilder: expand=expand, ) + @staticmethod + def _requires_negpip(plan: PromptControlSegmentPlan) -> bool: + """Return whether any cleaned positive or negative segment needs NegPiP.""" + + return any( + contains_negative_prompt_weight(chunk.text) + for side in (plan.positive, plan.negative) + for chunk in side.chunks + ) + def _sampling_inputs( self, *, diff --git a/simple_syrup/services/negpip_model_service.py b/simple_syrup/services/negpip_model_service.py new file mode 100644 index 0000000..09bd0bb --- /dev/null +++ b/simple_syrup/services/negpip_model_service.py @@ -0,0 +1,213 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Clone and patch supported MODEL/CLIP pairs for automatic NegPiP.""" + +# NegPiP behavior is adapted from ComfyUI-ppm and its credited predecessors. +# See third_party/manifest.toml and third_party/NOTICE.md. + +from __future__ import annotations + +import logging +from functools import partial + +from comfy.model_base import SDXL, Anima, BaseModel, Krea2, SDXLRefiner +from comfy.model_patcher import ModelPatcher +from comfy.sd import CLIP + +from ..runtime.clip_patcher_mutations import ( + ClipBooleanOptionMutation, + ClipCallableObjectPatchMutation, + ClipTokenizerMutation, +) +from ..runtime.model_patcher_mutations import ( + ModelAttentionPatchMutation, + ModelBooleanOptionMutation, + ModelCallableObjectPatchMutation, + ModelDiffusionWrapperMutation, + ModelInteropDiffusionWrapperMutation, +) +from ..runtime.negpip.anima import ( + WRAPPER_KEY as ANIMA_WRAPPER_KEY, +) +from ..runtime.negpip.anima import ( + anima_attn2_negpip, + anima_diffusion_negpip_wrapper, + anima_extra_conds_negpip_wrapper, +) +from ..runtime.negpip.krea2 import ( + CLIP_MARKER, + KREA_TOKEN_KEY, + Krea2NegpipTokenizer, + encode_krea2_token_weights_negpip, + krea2_attn1_negpip, + krea2_diffusion_negpip_wrapper, + krea2_extra_conds_negpip_wrapper, +) +from ..runtime.negpip.krea2 import ( + WRAPPER_KEY as KREA_WRAPPER_KEY, +) +from ..runtime.negpip.standard import ( + encode_token_weights_negpip, + standard_attn2_negpip, +) +from ..runtime.patcher_lifecycle import PATCHER_LIFECYCLE + +LOGGER = logging.getLogger(__name__) +MODEL_MARKER = "ppm_negpip" +SUPPORTED_STANDARD_ENCODERS = ("clip_g", "clip_l", "t5xxl", "llama", "qwen3_06b") + + +class NegpipModelService: + """Apply exactly one family-specific NegPiP patch set when supported.""" + + def prepare(self, model: object, clip: object) -> tuple[object, object]: + """Return a patched clone pair or the original unsupported pair unchanged.""" + + if not isinstance(model, ModelPatcher) or not isinstance(clip, CLIP): + raise TypeError("Automatic NegPiP requires Comfy MODEL and CLIP objects.") + marker = model.model_options.get(MODEL_MARKER, False) + if not isinstance(marker, bool): + raise TypeError("MODEL ppm_negpip marker must be boolean.") + if marker: + LOGGER.debug("Automatic NegPiP reused an already-patched MODEL") + return model, clip + + model_type = type(model.model) + if model_type is Krea2: + return self._prepare_krea2(model, clip) + if model_type is Anima: + return self._prepare_anima(model, clip) + if model_type is BaseModel or issubclass(model_type, (SDXL, SDXLRefiner)): + return self._prepare_standard(model, clip) + LOGGER.debug( + "Automatic NegPiP skipped unsupported model family", + extra={"model_type": model_type.__qualname__}, + ) + return model, clip + + def _prepare_standard( + self, + model: ModelPatcher, + clip: CLIP, + ) -> tuple[ModelPatcher, CLIP]: + """Install PPM-compatible interleaved key/value encoding on SD1 or SDXL.""" + + encoders = [ + name + for name in SUPPORTED_STANDARD_ENCODERS + if hasattr(clip.patcher.model, name) + ] + if not encoders: + LOGGER.warning("Automatic NegPiP found no supported standard text encoder") + return model, clip + prepared_clip = PATCHER_LIFECYCLE.derive_clip( + clip, + ( + *( + ClipCallableObjectPatchMutation( + f"{encoder_name}.encode_token_weights", + partial( + encode_token_weights_negpip, + getattr(clip.patcher.model, encoder_name), + ), + ) + for encoder_name in encoders + ), + ClipBooleanOptionMutation(MODEL_MARKER, True), + ), + operation="automatic standard NegPiP CLIP preparation", + ) + prepared_model = PATCHER_LIFECYCLE.derive_model( + model, + ( + ModelAttentionPatchMutation("attn2", standard_attn2_negpip), + ModelBooleanOptionMutation(MODEL_MARKER, True), + ), + operation="automatic standard NegPiP MODEL preparation", + ) + return prepared_model, prepared_clip + + def _prepare_anima( + self, + model: ModelPatcher, + clip: CLIP, + ) -> tuple[ModelPatcher, CLIP]: + """Install PPM-compatible Anima weight-mask conditions and attention.""" + + previous = model.get_model_object("extra_conds") + prepared_model = PATCHER_LIFECYCLE.derive_model( + model, + ( + ModelCallableObjectPatchMutation( + "extra_conds", + anima_extra_conds_negpip_wrapper(previous), + ), + ModelInteropDiffusionWrapperMutation( + ANIMA_WRAPPER_KEY, + anima_diffusion_negpip_wrapper, + ), + ModelAttentionPatchMutation("attn2", anima_attn2_negpip), + ModelBooleanOptionMutation(MODEL_MARKER, True), + ), + operation="automatic Anima NegPiP MODEL preparation", + ) + prepared_clip = PATCHER_LIFECYCLE.derive_clip( + clip, + (ClipBooleanOptionMutation(MODEL_MARKER, True),), + operation="automatic Anima NegPiP CLIP preparation", + ) + return prepared_model, prepared_clip + + def _prepare_krea2( + self, + model: ModelPatcher, + clip: CLIP, + ) -> tuple[ModelPatcher, CLIP]: + """Install Krea's shape-preserving sign-mask encoder and attention patch.""" + + if not hasattr(clip.patcher.model, KREA_TOKEN_KEY): + LOGGER.warning("Automatic NegPiP found no Krea Qwen3-VL text encoder") + return model, clip + outer_encoder = clip.patcher.get_model_object("encode_token_weights") + prepared_clip = PATCHER_LIFECYCLE.derive_clip( + clip, + ( + ClipTokenizerMutation( + clip.tokenizer, + Krea2NegpipTokenizer(clip.tokenizer), + ), + ClipCallableObjectPatchMutation( + "encode_token_weights", + partial(encode_krea2_token_weights_negpip, outer_encoder), + ), + ClipBooleanOptionMutation(CLIP_MARKER, True), + ), + operation="automatic Krea 2 NegPiP CLIP preparation", + ) + previous = model.get_model_object("extra_conds") + prepared_model = PATCHER_LIFECYCLE.derive_model( + model, + ( + ModelCallableObjectPatchMutation( + "extra_conds", + krea2_extra_conds_negpip_wrapper(previous), + ), + ModelDiffusionWrapperMutation( + KREA_WRAPPER_KEY, + krea2_diffusion_negpip_wrapper, + ), + ModelAttentionPatchMutation("attn1", krea2_attn1_negpip), + ModelBooleanOptionMutation(MODEL_MARKER, True), + ), + operation="automatic Krea 2 NegPiP MODEL preparation", + ) + LOGGER.info( + "Automatic NegPiP enabled", + extra={"model_family": "krea2", "encoder": KREA_TOKEN_KEY}, + ) + return prepared_model, prepared_clip + + +NEGPIP_MODEL_SERVICE = NegpipModelService() diff --git a/tests/test_negative_prompt_weights.py b/tests/test_negative_prompt_weights.py new file mode 100644 index 0000000..1887426 --- /dev/null +++ b/tests/test_negative_prompt_weights.py @@ -0,0 +1,61 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Verify automatic NegPiP trigger detection.""" + +from __future__ import annotations + +import pytest + +from simple_syrup.domain.negative_prompt_weights import ( + contains_negative_prompt_weight, +) + + +@pytest.mark.parametrize( + "text", + [ + "(1girl:-2.00)", + "portrait, (red jacket: -1.5)", + "((nested):-0.25)", + "[plain:(scheduled concept:-1.0):0.5]", + "outer ((inner):-3.0)", + ], +) +def test_detector_admits_effective_negative_prompt_weights(text: str) -> None: + """Recognize valid negative emphasis wherever Prompt Control may schedule it.""" + + assert contains_negative_prompt_weight(text) is True + + +@pytest.mark.parametrize( + "text", + [ + "1girl:-2.00", + "(1girl:2.00)", + "(1girl)", + "(1girl:not-a-number)", + r"escaped \(1girl:-2.0\)", + "unfinished (1girl:-2.0", + "STYLE(A1111, length)", + "", + ], +) +def test_detector_rejects_non_negative_weight_syntax(text: str) -> None: + """Do not activate for plain text, positive weights, escapes, or malformed input.""" + + assert contains_negative_prompt_weight(text) is False + + +def test_detector_resolves_nested_effective_weight() -> None: + """An inner explicit positive weight overrides a negative outer emphasis.""" + + assert contains_negative_prompt_weight("((kept positive:2.0):-3.0)") is False + + +def test_detector_requires_text() -> None: + """Reject dynamic non-text values before prompt planning.""" + + with pytest.raises(TypeError, match="requires text"): + contains_negative_prompt_weight(object()) # type: ignore[arg-type] diff --git a/tests/test_negpip_integration_workflow.py b/tests/test_negpip_integration_workflow.py new file mode 100644 index 0000000..85994cb --- /dev/null +++ b/tests/test_negpip_integration_workflow.py @@ -0,0 +1,188 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Verify isolated automatic NegPiP proof workflow construction.""" + +from __future__ import annotations + +from typing import cast + +import pytest + +from tools.negpip_integration.workflow import ( + REFINER_BASE_PROMPT, + REFINER_SWITCH_STEP, + NegpipFixtureSelections, + NegpipLiveFamily, + NegpipLiveWorkflowBuilder, +) + + +@pytest.fixture +def builder() -> NegpipLiveWorkflowBuilder: + """Return a builder with deterministic nested Comfy selections.""" + + return NegpipLiveWorkflowBuilder( + NegpipFixtureSelections( + sd1_checkpoint=r"proof\sd1.safetensors", + sdxl_checkpoint=r"proof\sdxl.safetensors", + sdxl_refiner_checkpoint=r"proof\sdxl-refiner.safetensors", + anima_diffusion=r"proof\anima.safetensors", + anima_text_encoder=r"proof\anima-te.safetensors", + krea2_diffusion=r"proof\krea2.safetensors", + krea2_text_encoder=r"proof\krea2-te.safetensors", + qwen_image_vae=r"proof\qwen-image-vae.safetensors", + ) + ) + + +@pytest.mark.parametrize("family", tuple(NegpipLiveFamily)) +def test_triggered_workflow_samples_and_requires_runtime_evidence( + builder: NegpipLiveWorkflowBuilder, + family: NegpipLiveFamily, +) -> None: + """Every family uses the public node and synchronized callback observer.""" + + built = builder.build(family, run_id=f"proof:{family.value}", trigger=True) + class_types = [str(node["class_type"]) for node in built.prompt.values()] + schedule = next( + node + for node in built.prompt.values() + if node["class_type"] == "SimpleSyrup.ScheduleAndEncodePromptsWithPromptControl" + ) + + inputs = cast(dict[str, object], schedule["inputs"]) + assert "bright (red:-1.0) jacket" in str(inputs["positive_prompt"]) + expected_sampler = ( + "KSamplerAdvanced" if family is NegpipLiveFamily.SDXL_REFINER else "KSampler" + ) + assert expected_sampler in class_types + assert "VAEDecode" in class_types + assert "SaveImage" in class_types + assert "SimpleSyrupBenchmark.InstrumentNegpipModel" in class_types + assert "SimpleSyrupBenchmark.ReadNegpipRuntime" in class_types + assert built.runtime_node_id is not None + assert built.conditioning_node_id is not None + + +@pytest.mark.parametrize("family", tuple(NegpipLiveFamily)) +def test_control_workflow_proves_automatic_gate_stays_off( + builder: NegpipLiveWorkflowBuilder, + family: NegpipLiveFamily, +) -> None: + """A no-negative-weight control saves a sampled unmodified image.""" + + built = builder.build(family, run_id=f"control:{family.value}", trigger=False) + class_types = [str(node["class_type"]) for node in built.prompt.values()] + + assert "SimpleSyrupBenchmark.SnapshotModelModifier" in class_types + expected_sampler = ( + "KSamplerAdvanced" if family is NegpipLiveFamily.SDXL_REFINER else "KSampler" + ) + assert expected_sampler in class_types + assert "VAEDecode" in class_types + assert "SaveImage" in class_types + assert "SimpleSyrupBenchmark.InstrumentNegpipModel" not in class_types + assert built.runtime_node_id is None + assert built.conditioning_node_id is None + + +def test_family_workflows_select_native_loaders_and_latents( + builder: NegpipLiveWorkflowBuilder, +) -> None: + """Use real family-specific loader and latent contracts.""" + + expected = { + NegpipLiveFamily.SD1: {"CheckpointLoaderSimple", "EmptyLatentImage"}, + NegpipLiveFamily.SDXL: {"CheckpointLoaderSimple", "EmptyLatentImage"}, + NegpipLiveFamily.SDXL_REFINER: { + "CheckpointLoaderSimple", + "EmptyLatentImage", + }, + NegpipLiveFamily.ANIMA: { + "SimpleSyrup.SimpleLoadAnima", + "EmptyCosmosLatentVideo", + }, + NegpipLiveFamily.KREA2: { + "UNETLoader", + "CLIPLoader", + "VAELoader", + "EmptySD3LatentImage", + }, + } + + for family, required in expected.items(): + built = builder.build(family, run_id=family.value, trigger=True) + class_types = {str(node["class_type"]) for node in built.prompt.values()} + assert required.issubset(class_types) + + +def test_refiner_workflow_runs_base_then_refiner_sampling( + builder: NegpipLiveWorkflowBuilder, +) -> None: + """Use SDXL base for high noise and the probed refiner for low noise.""" + + built = builder.build( + NegpipLiveFamily.SDXL_REFINER, + run_id="refiner", + trigger=True, + ) + samplers = [ + node + for node in built.prompt.values() + if node["class_type"] == "KSamplerAdvanced" + ] + + assert len(samplers) == 2 + base_inputs = cast(dict[str, object], samplers[0]["inputs"]) + refiner_inputs = cast(dict[str, object], samplers[1]["inputs"]) + assert base_inputs["add_noise"] == "enable" + assert base_inputs["end_at_step"] == REFINER_SWITCH_STEP + assert base_inputs["return_with_leftover_noise"] == "enable" + assert refiner_inputs["add_noise"] == "disable" + assert refiner_inputs["start_at_step"] == REFINER_SWITCH_STEP + assert refiner_inputs["end_at_step"] == 24 + base_positive = next( + node + for node in built.prompt.values() + if node["class_type"] == "CLIPTextEncode" + and cast(dict[str, object], node["inputs"])["text"] == REFINER_BASE_PROMPT + ) + base_text = cast(dict[str, object], base_positive["inputs"])["text"] + assert isinstance(base_text, str) + assert "red" not in base_text + + +def test_ppm_baseline_prepatches_the_same_schedule_path( + builder: NegpipLiveWorkflowBuilder, +) -> None: + """Put pinned PPM before Schedule & Encode as the behavioral oracle.""" + + built = builder.build( + NegpipLiveFamily.SD1, + run_id="ppm-baseline", + trigger=True, + baseline_ppm=True, + ) + class_types = [str(node["class_type"]) for node in built.prompt.values()] + + assert built.mode == "ppm_baseline" + assert class_types.count("CLIPNegPip") == 1 + assert ( + class_types.count("SimpleSyrup.ScheduleAndEncodePromptsWithPromptControl") == 1 + ) + + +def test_ppm_baseline_rejects_an_untriggered_workflow( + builder: NegpipLiveWorkflowBuilder, +) -> None: + """Keep ordinary controls free from every NegPiP patch.""" + + with pytest.raises(ValueError, match="requires a negative weight"): + builder.build( + NegpipLiveFamily.SD1, + run_id="invalid", + trigger=False, + baseline_ppm=True, + ) diff --git a/tests/test_negpip_model_service.py b/tests/test_negpip_model_service.py new file mode 100644 index 0000000..58e4e3c --- /dev/null +++ b/tests/test_negpip_model_service.py @@ -0,0 +1,192 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Verify automatic NegPiP dispatch on real Comfy patcher objects.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any, cast + +import pytest +import torch +from comfy.model_base import SDXL, Anima, BaseModel, Krea2, SDXLRefiner +from comfy.model_patcher import ModelPatcher +from comfy.patcher_extension import WrappersMP +from comfy.sd import CLIP + +from simple_syrup.runtime.negpip.anima import ( + WRAPPER_KEY as ANIMA_WRAPPER_KEY, +) +from simple_syrup.runtime.negpip.anima import ( + anima_attn2_negpip, +) +from simple_syrup.runtime.negpip.krea2 import ( + CLIP_MARKER, + KREA_TOKEN_KEY, + Krea2NegpipTokenizer, + krea2_attn1_negpip, +) +from simple_syrup.runtime.negpip.krea2 import ( + WRAPPER_KEY as KREA_WRAPPER_KEY, +) +from simple_syrup.runtime.negpip.standard import standard_attn2_negpip +from simple_syrup.services.negpip_model_service import ( + MODEL_MARKER, + NegpipModelService, +) + + +class _Encoder(torch.nn.Module): + """Expose the encoder method patched by the standard NegPiP path.""" + + def encode_token_weights(self, pairs: object) -> object: + """Return the supplied placeholder pairs.""" + + return pairs + + +class _ClipRoot(torch.nn.Module): + """Provide the model object structure used by supported CLIP families.""" + + def __init__(self, *, krea: bool = False) -> None: + """Install either the standard or Krea encoder surface.""" + + super().__init__() + if krea: + setattr(self, KREA_TOKEN_KEY, _Encoder()) + else: + self.clip_l = _Encoder() + + def encode_token_weights( + self, + pairs: object, + *, + template_end: int = -1, + ) -> tuple[object, None, dict[str, object]]: + """Stand in for Krea's root shape-preserving encoder.""" + + del template_end + return pairs, None, {} + + +class _Tokenizer: + """Represent the installed tokenizer retained by a Krea proxy.""" + + +@pytest.mark.parametrize("model_class", (BaseModel, SDXL, SDXLRefiner)) +def test_service_patches_every_standard_ppm_family( + model_class: type[BaseModel], +) -> None: + """SD1, SDXL, and SDXL Refiner receive one cloned PPM-equivalent path.""" + + model = _model_patcher(model_class) + clip = _clip(krea=False) + + prepared_model, prepared_clip = NegpipModelService().prepare(model, clip) + + assert isinstance(prepared_model, ModelPatcher) + assert isinstance(prepared_clip, CLIP) + assert prepared_model is not model + assert prepared_clip is not clip + assert MODEL_MARKER not in model.model_options + assert MODEL_MARKER not in clip.patcher.model_options + assert prepared_model.model_options[MODEL_MARKER] is True + assert prepared_clip.patcher.model_options[MODEL_MARKER] is True + assert _attention_patch(prepared_model, "attn2_patch") is standard_attn2_negpip + assert "clip_l.encode_token_weights" in prepared_clip.patcher.object_patches + + +def test_service_patches_anima_with_mask_wrapper_and_attention() -> None: + """Anima receives its extra condition, diffusion wrapper, and V patch.""" + + prepared_model, prepared_clip = NegpipModelService().prepare( + _model_patcher(Anima), + _clip(krea=False), + ) + + model = cast(ModelPatcher, prepared_model) + clip = cast(CLIP, prepared_clip) + assert model.model_options[MODEL_MARKER] is True + assert clip.patcher.model_options[MODEL_MARKER] is True + assert "extra_conds" in model.object_patches + assert _attention_patch(model, "attn2_patch") is anima_attn2_negpip + assert ANIMA_WRAPPER_KEY in model.wrappers[WrappersMP.DIFFUSION_MODEL] + + +def test_service_patches_krea_without_claiming_ppm_clip_encoding() -> None: + """Krea uses its layered encoder proxy and joint attn1 value patch.""" + + source_clip = _clip(krea=True) + prepared_model, prepared_clip = NegpipModelService().prepare( + _model_patcher(Krea2), + source_clip, + ) + + model = cast(ModelPatcher, prepared_model) + clip = cast(CLIP, prepared_clip) + assert model.model_options[MODEL_MARKER] is True + assert MODEL_MARKER not in clip.patcher.model_options + assert clip.patcher.model_options[CLIP_MARKER] is True + assert isinstance(clip.tokenizer, Krea2NegpipTokenizer) + assert clip.tokenizer is not source_clip.tokenizer + assert "encode_token_weights" in clip.patcher.object_patches + assert "extra_conds" in model.object_patches + assert _attention_patch(model, "attn1_patch") is krea2_attn1_negpip + assert KREA_WRAPPER_KEY in model.wrappers[WrappersMP.DIFFUSION_MODEL] + + +def test_service_reuses_already_patched_pair_without_double_patching() -> None: + """An existing PPM model marker makes automatic preparation idempotent.""" + + model = _model_patcher(BaseModel) + clip = _clip(krea=False) + model.model_options[MODEL_MARKER] = True + + prepared_model, prepared_clip = NegpipModelService().prepare(model, clip) + + assert prepared_model is model + assert prepared_clip is clip + prepared_model_typed = cast(ModelPatcher, prepared_model) + transformer_options = cast( + dict[str, object], + prepared_model_typed.model_options["transformer_options"], + ) + assert "patches" not in transformer_options + + +def _model_patcher(model_class: type[BaseModel]) -> ModelPatcher: + """Construct an unloaded family instance behind Comfy's real patcher.""" + + model = object.__new__(model_class) + torch.nn.Module.__init__(model) + device = torch.device("cpu") + return ModelPatcher(model, load_device=device, offload_device=device) + + +def _clip(*, krea: bool) -> CLIP: + """Construct a cloneable unloaded CLIP around a real model patcher.""" + + clip = CLIP(no_init=True) + root = _ClipRoot(krea=krea) + device = torch.device("cpu") + clip.patcher = ModelPatcher(root, load_device=device, offload_device=device) + clip.cond_stage_model = root + clip.tokenizer = _Tokenizer() + clip.layer_idx = None + clip.tokenizer_options = {} + clip.use_clip_schedule = False + clip.apply_hooks_to_conds = None + return clip + + +def _attention_patch(model: ModelPatcher, key: str) -> Callable[..., Any]: + """Return the single installed attention patch from model options.""" + + transformer_options = cast( + dict[str, object], model.model_options["transformer_options"] + ) + patches = cast(dict[str, list[Callable[..., Any]]], transformer_options["patches"]) + assert len(patches[key]) == 1 + return patches[key][0] diff --git a/tests/test_negpip_runtime.py b/tests/test_negpip_runtime.py new file mode 100644 index 0000000..816f626 --- /dev/null +++ b/tests/test_negpip_runtime.py @@ -0,0 +1,291 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Prove family-specific NegPiP tensor and wrapper invariants.""" + +from __future__ import annotations + +from typing import Any, cast + +import pytest +import torch + +from simple_syrup.runtime.negpip.anima import ( + CONDITION_MASK_KEY as ANIMA_CONDITION_MASK_KEY, +) +from simple_syrup.runtime.negpip.anima import ( + TRANSFORMER_MASK_KEY as ANIMA_TRANSFORMER_MASK_KEY, +) +from simple_syrup.runtime.negpip.anima import ( + anima_attn2_negpip, + anima_diffusion_negpip_wrapper, + anima_extra_conds_negpip_wrapper, +) +from simple_syrup.runtime.negpip.krea2 import ( + CONDITION_MASK_KEY as KREA_CONDITION_MASK_KEY, +) +from simple_syrup.runtime.negpip.krea2 import ( + ENCODER_MASK_KEY, + encode_krea2_token_weights_negpip, + krea2_attn1_negpip, + krea2_diffusion_negpip_wrapper, + krea2_extra_conds_negpip_wrapper, +) +from simple_syrup.runtime.negpip.krea2 import ( + TRANSFORMER_MASK_KEY as KREA_TRANSFORMER_MASK_KEY, +) +from simple_syrup.runtime.negpip.standard import ( + encode_token_weights_negpip, + standard_attn2_negpip, +) + + +class _StandardEncoder: + """Produce deterministic token and empty-prompt embeddings.""" + + special_tokens: dict[str, int] = {} + + def gen_empty_tokens( + self, + special_tokens: dict[str, int], + length: int, + ) -> list[int]: + """Return a fixed empty token row matching the requested length.""" + + del special_tokens + return [0] * length + + def encode(self, sections: list[list[object]]) -> tuple[torch.Tensor, None]: + """Map source tokens to scalar embeddings and empty tokens to one.""" + + rows = [ + [[1.0 if token == 0 else float(cast(int, token))] for token in section] + for section in sections + ] + return torch.tensor(rows), None + + +def test_standard_negpip_interleaves_magnitude_keys_and_signed_values() -> None: + """Standard encoding doubles tokens and signs only the value positions.""" + + encoded, pooled = encode_token_weights_negpip( + cast(Any, _StandardEncoder()), + [[(3, -2.0), (5, 0.5)]], + ) + + assert pooled is None + assert isinstance(encoded, torch.Tensor) + assert encoded.flatten().tolist() == [5.0, -5.0, 3.0, 3.0] + + query = torch.tensor([[[9.0], [8.0]]]) + key = encoded.clone() + value = encoded.clone() + prepared_query, prepared_key, prepared_value = standard_attn2_negpip( + query, + key, + value, + {}, + ) + assert prepared_query is query + assert prepared_key.flatten().tolist() == [5.0, 3.0] + assert prepared_value.flatten().tolist() == [-5.0, 3.0] + + +def test_anima_negpip_preserves_magnitude_and_propagates_value_mask() -> None: + """Anima moves signs through conditions and changes only attention values.""" + + observed_weights: list[torch.Tensor] = [] + + def base_extra_conds(**kwargs: object) -> dict[str, object]: + weights = kwargs["t5xxl_weights"] + assert isinstance(weights, torch.Tensor) + observed_weights.append(weights) + return {"base": "condition"} + + wrapped = anima_extra_conds_negpip_wrapper(base_extra_conds) + output = wrapped(t5xxl_weights=torch.tensor([-2.0, 0.5, 1.0])) + + assert torch.equal(observed_weights[0], torch.tensor([2.0, 0.5, 1.0])) + condition = output[ANIMA_CONDITION_MASK_KEY] + multiplier = cast(Any, condition).cond + assert multiplier.shape == (1, 512, 1) + assert multiplier[0, :3, 0].tolist() == [-1.0, 1.0, 1.0] + assert torch.all(multiplier[0, 3:, 0] == 1.0) + + captured: dict[str, object] = {} + + def executor(*args: object, **kwargs: object) -> str: + del args + captured.update(kwargs) + return "executed" + + context = torch.zeros((1, 512, 4)) + result = anima_diffusion_negpip_wrapper( + executor, + object(), + object(), + context, + transformer_options={"existing": True}, + **{ANIMA_CONDITION_MASK_KEY: multiplier}, + ) + assert result == "executed" + options = cast(dict[str, object], captured["transformer_options"]) + assert options["existing"] is True + assert torch.equal( + cast(torch.Tensor, options[ANIMA_TRANSFORMER_MASK_KEY]), multiplier + ) + + query = torch.ones((1, 1, 3, 1)) + key = torch.full_like(query, 2.0) + value = torch.tensor([[[[3.0], [4.0], [5.0]]]]) + attention = anima_attn2_negpip( + query, + key, + value, + extra_options={ANIMA_TRANSFORMER_MASK_KEY: multiplier[:, :3]}, + ) + assert attention["q"] is query + assert attention["k"] is key + assert cast(torch.Tensor, attention["v"]).flatten().tolist() == [ + -3.0, + 4.0, + 5.0, + ] + + +def test_krea2_negpip_preserves_shape_and_signs_only_text_values() -> None: + """Krea retains layered encoding and leaves Q, K, and image V untouched.""" + + tokens: dict[str, list[list[tuple[object, ...]]]] = { + "qwen3vl_4b": [ + [ + (151644, 1.0), + (0, 1.0), + (198, 1.0), + (151644, 1.0), + (872, 1.0), + (198, 1.0), + (10, -2.0), + (11, 0.5), + ] + ] + } + observed: dict[str, object] = {} + + def original( + prepared: dict[str, list[list[tuple[object, ...]]]], + *, + template_end: int, + ) -> tuple[torch.Tensor, None, dict[str, object]]: + observed["tokens"] = prepared + observed["template_end"] = template_end + return torch.ones((1, 2, 30_720)), None, {"source": True} + + conditioning, pooled, extra = encode_krea2_token_weights_negpip( + original, + tokens, + ) + + conditioning_tensor = cast(torch.Tensor, conditioning) + assert conditioning_tensor.shape == (1, 2, 30_720) + assert pooled is None + absolute = cast( + dict[str, list[list[tuple[object, ...]]]], + observed["tokens"], + ) + assert [pair[1] for pair in absolute["qwen3vl_4b"][0][-2:]] == [2.0, 0.5] + metadata = cast(dict[str, object], extra) + multiplier = cast(torch.Tensor, metadata[ENCODER_MASK_KEY]) + assert multiplier.flatten().tolist() == [-1.0, 1.0] + + wrapped_extra = krea2_extra_conds_negpip_wrapper(lambda **kwargs: {}) + processed = wrapped_extra(**{ENCODER_MASK_KEY: multiplier}) + condition = processed[KREA_CONDITION_MASK_KEY] + processed_multiplier = cast(Any, condition).cond + + captured: dict[str, object] = {} + + def executor(*args: object, **kwargs: object) -> str: + del args + captured.update(kwargs) + return "executed" + + assert ( + krea2_diffusion_negpip_wrapper( + executor, + transformer_options={"img_slice": [2, 4]}, + **{KREA_CONDITION_MASK_KEY: processed_multiplier}, + ) + == "executed" + ) + options = cast(dict[str, Any], captured["transformer_options"]) + assert options["img_slice"] == [2, 4] + + positional_capture: dict[str, object] = {} + + def positional_executor(*args: object, **kwargs: object) -> str: + positional_capture["args"] = args + positional_capture["kwargs"] = kwargs + return "positional" + + positional_options = {"img_slice": [2, 4]} + assert ( + krea2_diffusion_negpip_wrapper( + positional_executor, + object(), + object(), + object(), + None, + None, + positional_options, + **{KREA_CONDITION_MASK_KEY: processed_multiplier}, + ) + == "positional" + ) + positional_args = cast(tuple[object, ...], positional_capture["args"]) + prepared_positional = cast(dict[str, object], positional_args[5]) + assert prepared_positional is not positional_options + assert prepared_positional[KREA_TRANSFORMER_MASK_KEY] is processed_multiplier + assert "transformer_options" not in cast( + dict[str, object], positional_capture["kwargs"] + ) + + query = torch.arange(8.0).reshape(1, 1, 4, 2) + key = query + 10.0 + value = query + 20.0 + attention = krea2_attn1_negpip( + query, + key, + value, + extra_options=options, + ) + assert attention["q"] is query + assert attention["k"] is key + prepared_value = cast(torch.Tensor, attention["v"]) + assert prepared_value[0, 0, 0].tolist() == [-20.0, -21.0] + assert prepared_value[0, 0, 1].tolist() == [22.0, 23.0] + assert torch.equal(prepared_value[:, :, 2:], value[:, :, 2:]) + assert torch.equal(value, query + 20.0) + + +@pytest.mark.parametrize( + "image_slice", + (None, [3, 4]), +) +def test_krea2_negpip_rejects_unprovable_text_boundaries( + image_slice: object, +) -> None: + """Krea fails closed when the model boundary cannot align to its sign mask.""" + + options: dict[str, object] = {KREA_TRANSFORMER_MASK_KEY: torch.ones((1, 2, 1))} + if image_slice is not None: + options["img_slice"] = image_slice + + with pytest.raises(ValueError, match="boundary|does not match"): + krea2_attn1_negpip( + torch.ones((1, 1, 4, 1)), + torch.ones((1, 1, 4, 1)), + torch.ones((1, 1, 4, 1)), + extra_options=options, + ) diff --git a/tests/test_negpip_runtime_probe.py b/tests/test_negpip_runtime_probe.py new file mode 100644 index 0000000..1c49956 --- /dev/null +++ b/tests/test_negpip_runtime_probe.py @@ -0,0 +1,133 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Verify benchmark-only live NegPiP callback introspection.""" + +from __future__ import annotations + +import torch + +from simple_syrup.runtime.negpip.krea2 import TRANSFORMER_MASK_KEY +from tools.attention_coupling_benchmark.comfy_probe.negpip_runtime import ( + InstrumentNegpipModelV3, + ReadNegpipRuntimeV3, + _NegpipProbeState, + _observe_masked, + _observe_standard, + _owned_negpip_callback, +) + + +def test_negpip_runtime_probe_schemas_are_stable() -> None: + """Expose distinct instrument and synchronized evidence node IDs.""" + + instrument = InstrumentNegpipModelV3.define_schema() + reader = ReadNegpipRuntimeV3.define_schema() + + assert instrument.node_id == "SimpleSyrupBenchmark.InstrumentNegpipModel" + assert reader.node_id == "SimpleSyrupBenchmark.ReadNegpipRuntime" + + +def test_standard_probe_validates_live_interleaved_selection() -> None: + """Record one standard callback only after exact even/odd selection.""" + + state = _NegpipProbeState("standard", "attn2_patch", "callback") + query = torch.ones((1, 2, 1)) + key = torch.tensor([[[1.0], [1.0], [2.0], [2.0]]]) + value = torch.tensor([[[1.0], [-1.0], [2.0], [2.0]]]) + + _observe_standard( + state, + query, + key, + value, + (query, key[:, 0::2], value[:, 1::2]), + ) + + assert state.attention_calls == 1 + assert state.negative_mask_calls == 1 + assert state.input_value_shape == [1, 4, 1] + assert state.output_value_shape == [1, 2, 1] + assert state.negative_token_count == 1 + assert state.negative_token_positions == [0] + assert state.negative_token_locations == [[0, 0]] + + +def test_standard_probe_finds_signed_tokens_outside_cfg_batch_zero() -> None: + """Inspect every CFG row rather than assuming the positive prompt is first.""" + + state = _NegpipProbeState("standard", "attn2_patch", "callback") + query = torch.ones((2, 2, 1)) + key = torch.tensor( + [ + [[1.0], [1.0], [2.0], [2.0]], + [[1.0], [1.0], [2.0], [2.0]], + ] + ) + value = torch.tensor( + [ + [[1.0], [1.0], [2.0], [2.0]], + [[1.0], [1.0], [2.0], [-2.0]], + ] + ) + + _observe_standard( + state, + query, + key, + value, + (query, key[:, 0::2], value[:, 1::2]), + ) + + assert state.negative_mask_calls == 1 + assert state.negative_token_count == 1 + assert state.negative_token_positions == [1] + assert state.negative_token_locations == [[1, 1]] + + +def test_probe_admits_the_pinned_ppm_standard_callback_identity() -> None: + """Recognize PPM even when Comfy prefixes its module with a Windows path.""" + + def sdxl_attn2_negpip() -> None: + """Stand in for the identity-checked pinned PPM callback.""" + + sdxl_attn2_negpip.__module__ = "managed_comfyui_ppm.src.negpip.unet_negpip" + sdxl_attn2_negpip.__qualname__ = "sdxl_attn2_negpip" + + patch_name, family, callback = _owned_negpip_callback( + {"attn2_patch": [sdxl_attn2_negpip]} + ) + + assert patch_name == "attn2_patch" + assert family == "standard" + assert callback is sdxl_attn2_negpip + + +def test_krea_probe_validates_text_only_value_signing() -> None: + """Record Krea only when its image suffix remains exact.""" + + state = _NegpipProbeState("krea2", "attn1_patch", "callback") + query = torch.ones((1, 1, 4, 1)) + key = torch.ones((1, 1, 4, 1)) * 2 + value = torch.tensor([[[[3.0], [4.0], [5.0], [6.0]]]]) + multiplier = torch.tensor([[[-1.0], [1.0]]]) + output_value = value.clone() + output_value[:, :, :2] *= multiplier.unsqueeze(1) + + _observe_masked( + state, + query, + key, + value, + {"q": query, "k": key, "v": output_value}, + {TRANSFORMER_MASK_KEY: multiplier, "img_slice": [2, 4]}, + ) + + assert state.attention_calls == 1 + assert state.negative_mask_calls == 1 + assert state.text_length == 2 + assert state.mask_shape == [1, 2, 1] + assert state.negative_token_count == 1 + assert state.negative_token_positions == [0] + assert state.negative_token_locations == [[0, 0]] diff --git a/tests/test_negpip_visual_proof.py b/tests/test_negpip_visual_proof.py new file mode 100644 index 0000000..74f614d --- /dev/null +++ b/tests/test_negpip_visual_proof.py @@ -0,0 +1,93 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Verify decoded NegPiP proof artifacts and contact-sheet evidence.""" + +from __future__ import annotations + +import io +from pathlib import Path + +from PIL import Image + +from tools.comfy_api import JsonObject +from tools.negpip_integration.visual_proof import NegpipVisualProofRecorder +from tools.negpip_integration.workflow import NegpipLiveFamily + + +def test_visual_recorder_persists_pairs_and_contact_sheet(tmp_path: Path) -> None: + """Every family receives originals, pixel deltas, and visible labels.""" + + recorder = NegpipVisualProofRecorder(tmp_path) + families: JsonObject = {} + for index, family in enumerate(NegpipLiveFamily): + control = recorder.record( + family, + "control", + _png_bytes((20 + index, 40, 60)), + ) + negative = recorder.record( + family, + "negative", + _png_bytes((120 + index, 40, 60)), + ) + families[family.value] = { + "control": {"image": control}, + "negative": { + "image": negative, + "runtime": { + "family": family.value, + "attention_calls": 3, + "negative_mask_calls": 3, + }, + }, + } + + sheet = recorder.finalize(families) + + assert (tmp_path / str(sheet["file"])).is_file() + assert sheet["width"] == 1346 + assert sheet["height"] == 2528 + for family in NegpipLiveFamily: + result = families[family.value] + assert isinstance(result, dict) + comparison = result["image_comparison"] + assert isinstance(comparison, dict) + assert comparison["changed_pixels"] == 512 * 512 + assert comparison["mean_absolute_rgb_delta"] > 0 + + +def test_visual_recorder_rejects_identical_pair(tmp_path: Path) -> None: + """A decoded image must visibly change in every supported family.""" + + recorder = NegpipVisualProofRecorder(tmp_path) + image = _png_bytes((20, 40, 60)) + families: JsonObject = {} + for family in NegpipLiveFamily: + families[family.value] = { + "control": {"image": recorder.record(family, "control", image)}, + "negative": { + "image": recorder.record(family, "negative", image), + "runtime": { + "family": family.value, + "attention_calls": 1, + "negative_mask_calls": 1, + }, + }, + } + + try: + recorder.finalize(families) + except ValueError as error: + assert "images are equal" in str(error) + else: + raise AssertionError("Identical NegPiP proof images must be rejected.") + + +def _png_bytes(color: tuple[int, int, int]) -> bytes: + """Return one deterministic 512-square PNG.""" + + stream = io.BytesIO() + Image.new("RGB", (512, 512), color).save(stream, format="PNG") + return stream.getvalue() diff --git a/tests/test_prompt_control_schedule_encode_graph.py b/tests/test_prompt_control_schedule_encode_graph.py index 600d4d8..3b04b5f 100644 --- a/tests/test_prompt_control_schedule_encode_graph.py +++ b/tests/test_prompt_control_schedule_encode_graph.py @@ -60,6 +60,63 @@ def test_schedule_encode_graph_builds_single_conditioning_outputs( ] +@pytest.mark.parametrize( + ("positive_prompt", "negative_prompt"), + [ + ("portrait of (1girl:-2.0)", "blur"), + ("portrait", "(blur:-0.5)"), + ("portrait [SEP] (hands:-1.2)", "blur"), + ("portrait [0:(eyes:-1.5):0.5]", "blur"), + ], +) +def test_schedule_encode_graph_injects_negpip_for_negative_weights( + monkeypatch: pytest.MonkeyPatch, + positive_prompt: str, + negative_prompt: str, +) -> None: + """Any effective negative segment weight prepares MODEL and CLIP first.""" + + calls = _install_fake_prompt_control(monkeypatch) + + output = PromptControlScheduleEncodeGraphBuilder().build( + model=["model", 0], + clip=["clip", 0], + positive_prompt=positive_prompt, + negative_prompt=negative_prompt, + ) + + assert output.expand is not None + preparation_nodes = [ + node + for node in output.expand.values() + if node["class_type"] == "SimpleSyrup.ApplyAutomaticNegpip" + ] + assert len(preparation_nodes) == 1 + assert calls["encode"] + assert all(call["clip"] != ["clip", 0] for call in calls["encode"]) + + +def test_schedule_encode_graph_does_not_inject_negpip_for_nonnegative_weights( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Ordinary and positive-weight prompts retain the existing graph path.""" + + _install_fake_prompt_control(monkeypatch) + + output = PromptControlScheduleEncodeGraphBuilder().build( + model=["model", 0], + clip=["clip", 0], + positive_prompt="portrait of (1girl:2.0)", + negative_prompt="blur", + ) + + assert output.expand is not None + assert not any( + node["class_type"] == "SimpleSyrup.ApplyAutomaticNegpip" + for node in output.expand.values() + ) + + def test_schedule_encode_graph_packs_both_sides_to_matched_segment_counts( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_regional_model_patch_interop.py b/tests/test_regional_model_patch_interop.py index a20c34b..b8036ca 100644 --- a/tests/test_regional_model_patch_interop.py +++ b/tests/test_regional_model_patch_interop.py @@ -253,6 +253,55 @@ def test_validator_admits_exact_anima_negpip_without_mutation() -> None: assert model.object_patches["extra_conds"] is extra_conds +def test_validator_admits_owned_standard_negpip_without_mutation() -> None: + """Retain the automatic node's owned split-K/V callback identity.""" + + from simple_syrup.runtime.negpip.standard import standard_attn2_negpip + + model = _patcher() + model.model_options["ppm_negpip"] = True + model.set_model_attn2_patch(standard_attn2_negpip) + + report = REGIONAL_MODEL_PATCH_INTEROP_VALIDATOR.validate( + model, + _capabilities(RegionalModelFamily.STANDARD_UNET), + ) + + assert report.negpip is not None + assert report.negpip.attention_patch is standard_attn2_negpip + + +def test_validator_admits_owned_anima_negpip_without_mutation() -> None: + """Retain the automatic node's complete owned Anima callback family.""" + + from simple_syrup.runtime.negpip.anima import ( + anima_attn2_negpip, + anima_diffusion_negpip_wrapper, + anima_extra_conds_negpip_wrapper, + ) + + model = _patcher() + model.model_options["ppm_negpip"] = True + model.set_model_attn2_patch(anima_attn2_negpip) + model.add_wrapper_with_key( + WrappersMP.DIFFUSION_MODEL, + "ppm_negpip_anima", + anima_diffusion_negpip_wrapper, + ) + model.add_object_patch( + "extra_conds", + anima_extra_conds_negpip_wrapper(lambda **kwargs: {}), + ) + + report = REGIONAL_MODEL_PATCH_INTEROP_VALIDATOR.validate( + model, + _capabilities(RegionalModelFamily.ANIMA), + ) + + assert report.negpip is not None + assert report.negpip.attention_patch is anima_attn2_negpip + + @pytest.mark.parametrize( ("family", "configure", "message"), [ diff --git a/tests/test_registration.py b/tests/test_registration.py index 8a82ca3..c626794 100644 --- a/tests/test_registration.py +++ b/tests/test_registration.py @@ -75,6 +75,7 @@ BASE_NODE_IDS = [ ] PROMPT_CONTROL_NODE_IDS = [ + "SimpleSyrup.ApplyAutomaticNegpip", "SimpleSyrup.AttachRegionalGlobalConditioning", "SimpleSyrup.EncodePromptBatchWithPromptControl", "SimpleSyrup.LabelRegionalLoraHooks", diff --git a/tests/test_third_party_vendoring_contract.py b/tests/test_third_party_vendoring_contract.py index a824216..01b6653 100644 --- a/tests/test_third_party_vendoring_contract.py +++ b/tests/test_third_party_vendoring_contract.py @@ -192,3 +192,32 @@ def test_notice_records_sampler_and_tiled_diffusion_provenance() -> None: assert "k-diffusion Euler ancestral sampler" in notice assert "Mixture of Diffusers and MultiDiffusion tiled diffusion behavior" in notice assert "regional prompt mask blending" in notice + + +def test_negpip_provenance_records_baseline_and_original_implementations() -> None: + """NegPiP should trace through PPM to both credited original projects.""" + + manifest = tomllib.loads( + (REPO_ROOT / "third_party" / "manifest.toml").read_text(encoding="utf-8") + ) + components = {component["name"]: component for component in manifest["component"]} + + negpip = components["NegPiP prompt weighting"] + license_path = REPO_ROOT / negpip["license_file"] + + assert negpip["license"] == "AGPL-3.0" + assert "GNU AFFERO GENERAL PUBLIC" in license_path.read_text(encoding="utf-8") + assert negpip["source"] == "https://github.com/pamparamm/ComfyUI-ppm" + assert negpip["revision"] == "6c6c360155cace9d7091306c1b8e26d9c7438620" + assert negpip["origin_sources"] == [ + "https://github.com/laksjdjf/cd-tuner_negpip-ComfyUI@" + "938b838546cf774dc8841000996552cef52cccf3", + "https://github.com/hako-mikan/sd-webui-negpip@" + "fb7151f327ae56195f08b30b70d459493dadedbb", + ] + assert negpip["vendored_files"] == [ + "simple_syrup/runtime/negpip/standard.py", + "simple_syrup/runtime/negpip/anima.py", + "simple_syrup/runtime/negpip/krea2.py", + "simple_syrup/services/negpip_model_service.py", + ] diff --git a/third_party/NOTICE.md b/third_party/NOTICE.md index 6eb4ffb..5909790 100644 --- a/third_party/NOTICE.md +++ b/third_party/NOTICE.md @@ -56,3 +56,20 @@ tagger ONNX models and `selected_tags.csv` files at runtime from Hugging Face. These model files are not vendored in this repository. The runtime catalog points to the corresponding `SmilingWolf/*` repositories and stores downloaded files in the user's ComfyUI model directory. + +## NegPiP prompt weighting + +SimpleSyrup adapts the AGPL-3.0 NegPiP implementation from +`pamparamm/ComfyUI-ppm` at revision +`6c6c360155cace9d7091306c1b8e26d9c7438620`. The standard SD1/SDXL and Anima +paths preserve PPM's ModelPatcher-based magnitude-key and signed-value +behavior. The Krea 2 path extends the same signed-value rule to Krea's layered +Qwen conditioning and joint text/image attention while preserving its native +conditioning shape. + +PPM credits the original ComfyUI port to +`laksjdjf/cd-tuner_negpip-ComfyUI`; SimpleSyrup records revision +`938b838546cf774dc8841000996552cef52cccf3`. That port credits the original +Automatic1111 WebUI implementation in `hako-mikan/sd-webui-negpip`; +SimpleSyrup records revision +`fb7151f327ae56195f08b30b70d459493dadedbb`. diff --git a/third_party/licenses/negpip.LICENSE.txt b/third_party/licenses/negpip.LICENSE.txt new file mode 100644 index 0000000..3972c30 --- /dev/null +++ b/third_party/licenses/negpip.LICENSE.txt @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) 2024 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/third_party/manifest.toml b/third_party/manifest.toml index d4a03af..b9f8383 100644 --- a/third_party/manifest.toml +++ b/third_party/manifest.toml @@ -133,3 +133,25 @@ vendored_files = [ "simple_syrup/runtime/tiled_sampling_validation.py", "simple_syrup/services/detail_segs_as_regions_service.py", ] + +[[component]] +name = "NegPiP prompt weighting" +license = "AGPL-3.0" +license_file = "third_party/licenses/negpip.LICENSE.txt" +source = "https://github.com/pamparamm/ComfyUI-ppm" +revision = "6c6c360155cace9d7091306c1b8e26d9c7438620" +source_paths = [ + "src/nodes_ppm/clip_negpip.py", + "src/negpip/unet_negpip.py", + "src/negpip/anima_negpip.py", +] +origin_sources = [ + "https://github.com/laksjdjf/cd-tuner_negpip-ComfyUI@938b838546cf774dc8841000996552cef52cccf3", + "https://github.com/hako-mikan/sd-webui-negpip@fb7151f327ae56195f08b30b70d459493dadedbb", +] +vendored_files = [ + "simple_syrup/runtime/negpip/standard.py", + "simple_syrup/runtime/negpip/anima.py", + "simple_syrup/runtime/negpip/krea2.py", + "simple_syrup/services/negpip_model_service.py", +] diff --git a/tools/attention_coupling_benchmark/comfy_probe/__init__.py b/tools/attention_coupling_benchmark/comfy_probe/__init__.py index 9f1c523..2cc47a2 100644 --- a/tools/attention_coupling_benchmark/comfy_probe/__init__.py +++ b/tools/attention_coupling_benchmark/comfy_probe/__init__.py @@ -21,6 +21,7 @@ from .latent_completion import CompleteLatentV3 from .lora_execution_probe import InstrumentLoraModelV3, ReadLoraMetricsV3 from .materialization_parity_node import CompareMaterializationParityV3 from .model_modifier_snapshot import SnapshotModelModifierV3 +from .negpip_runtime import InstrumentNegpipModelV3, ReadNegpipRuntimeV3 from .operator_profile import ProfileIndexedModelCallV3, ReadOperatorProfileV3 from .prompt_control_expansion import SnapshotPromptControlExpansionV3 from .prompt_control_runtime import ( @@ -70,6 +71,8 @@ class BenchmarkProbeExtension(_ComfyExtensionBase): ReadLoraMetricsV3, CompareMaterializationParityV3, SnapshotModelModifierV3, + InstrumentNegpipModelV3, + ReadNegpipRuntimeV3, SnapshotPromptControlV3, SnapshotPromptControlExpansionV3, InstrumentPromptControlModelV3, diff --git a/tools/attention_coupling_benchmark/comfy_probe/negpip_runtime.py b/tools/attention_coupling_benchmark/comfy_probe/negpip_runtime.py new file mode 100644 index 0000000..029d37e --- /dev/null +++ b/tools/attention_coupling_benchmark/comfy_probe/negpip_runtime.py @@ -0,0 +1,416 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Instrument live NegPiP attention callbacks without changing their results.""" + +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass, field +from importlib import import_module +from typing import TYPE_CHECKING, Any, cast + +import torch +from comfy.model_patcher import ModelPatcher + +from simple_syrup.runtime.negpip.anima import ( + TRANSFORMER_MASK_KEY as ANIMA_MASK_KEY, +) +from simple_syrup.runtime.negpip.krea2 import ( + TRANSFORMER_MASK_KEY as KREA_MASK_KEY, +) + +_comfy_api: Any = None +if TYPE_CHECKING: + + class _ComfyNodeBase: + """Type-checking base for benchmark-only Comfy v3 nodes.""" + + pass + +else: + _comfy_api = import_module("comfy_api.latest") + _ComfyNodeBase = _comfy_api.io.ComfyNode + +_comfy_io: Any = None if TYPE_CHECKING else _comfy_api.io + + +@dataclass +class _NegpipProbeState: + """Accumulate live callback evidence for one managed workflow.""" + + family: str + patch_name: str + callback_name: str + attention_calls: int = 0 + negative_mask_calls: int = 0 + invariant_failures: list[str] = field(default_factory=list) + input_value_shape: list[int] | None = None + output_value_shape: list[int] | None = None + mask_shape: list[int] | None = None + text_length: int | None = None + negative_token_count: int = 0 + negative_token_positions: list[int] = field(default_factory=list) + negative_token_locations: list[list[int]] = field(default_factory=list) + + +_STATES: dict[str, _NegpipProbeState] = {} +_STATE_LOCK = threading.Lock() + + +class InstrumentNegpipModelV3(_ComfyNodeBase): + """Wrap one installed NegPiP callback and validate live tensor semantics.""" + + @classmethod + def define_schema(cls) -> Any: + """Declare benchmark-only MODEL instrumentation.""" + + return _comfy_io.Schema( + node_id="SimpleSyrupBenchmark.InstrumentNegpipModel", + display_name="Benchmark Instrument NegPiP Model", + category="SimpleSyrup/Benchmark", + inputs=[ + _comfy_io.Model.Input("model"), + _comfy_io.String.Input("run_id"), + ], + outputs=[_comfy_io.Model.Output("model")], + is_dev_only=True, + ) + + @classmethod + def execute(cls, model: object, run_id: str) -> Any: + """Clone MODEL and replace its owned callback with an observing delegate.""" + + if not isinstance(model, ModelPatcher): + raise TypeError("NegPiP instrumentation requires a Comfy MODEL.") + if not isinstance(run_id, str) or not run_id: + raise ValueError("NegPiP instrumentation run ID must not be empty.") + if model.model_options.get("ppm_negpip") is not True: + raise ValueError("NegPiP instrumentation requires a patched MODEL.") + cloned = model.clone() + patches = _transformer_patches(cloned) + patch_name, family, callback = _owned_negpip_callback(patches) + state = _NegpipProbeState( + family=family, + patch_name=patch_name, + callback_name=f"{callback.__module__}.{callback.__qualname__}", + ) + with _STATE_LOCK: + if run_id in _STATES: + raise ValueError(f"NegPiP probe run is already active: {run_id!r}.") + _STATES[run_id] = state + + def observe( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + *args: object, + **kwargs: object, + ) -> object: + """Delegate one callback and record its exact family invariant.""" + + result = callback(query, key, value, *args, **kwargs) + options = _extra_options(args, kwargs) + try: + _observe_result( + state, + query=query, + key=key, + value=value, + result=result, + options=options, + ) + except (TypeError, ValueError) as error: + with _STATE_LOCK: + state.invariant_failures.append(str(error)) + return result + + replacement = list(patches[patch_name]) + replacement[replacement.index(callback)] = observe + patches[patch_name] = replacement + return _comfy_io.NodeOutput(cloned) + + +class ReadNegpipRuntimeV3(_ComfyNodeBase): + """Publish and enforce completed live NegPiP callback evidence.""" + + @classmethod + def define_schema(cls) -> Any: + """Declare a latent-synchronized evidence output.""" + + return _comfy_io.Schema( + node_id="SimpleSyrupBenchmark.ReadNegpipRuntime", + display_name="Benchmark Read NegPiP Runtime", + category="SimpleSyrup/Benchmark", + inputs=[ + _comfy_io.Latent.Input("latent"), + _comfy_io.String.Input("run_id"), + ], + outputs=[ + _comfy_io.Latent.Output("latent"), + _comfy_io.String.Output("evidence_json"), + ], + is_output_node=True, + is_dev_only=True, + ) + + @classmethod + def execute(cls, latent: dict[str, Any], run_id: str) -> Any: + """Require observed negative-mask execution and return stable evidence.""" + + if torch.cuda.is_available(): + torch.cuda.synchronize() + with _STATE_LOCK: + state = _STATES.pop(run_id, None) + if state is None: + raise ValueError(f"NegPiP probe run was not instrumented: {run_id!r}.") + if state.attention_calls < 1: + raise ValueError("NegPiP attention callback was not executed.") + if state.negative_mask_calls < 1: + raise ValueError("NegPiP callback never observed a negative token.") + if state.invariant_failures: + raise ValueError( + "NegPiP live tensor invariants failed: " + + "; ".join(state.invariant_failures[:3]) + ) + evidence = { + "run_id": run_id, + "family": state.family, + "patch_name": state.patch_name, + "callback_name": state.callback_name, + "attention_calls": state.attention_calls, + "negative_mask_calls": state.negative_mask_calls, + "input_value_shape": state.input_value_shape, + "output_value_shape": state.output_value_shape, + "mask_shape": state.mask_shape, + "text_length": state.text_length, + "negative_token_count": state.negative_token_count, + "negative_token_positions": state.negative_token_positions, + "negative_token_locations": state.negative_token_locations, + "invariant_failures": state.invariant_failures, + } + encoded = json.dumps(evidence, sort_keys=True, separators=(",", ":")) + return _comfy_io.NodeOutput( + latent, + encoded, + ui={"negpip_runtime_evidence": [evidence]}, + ) + + +def _transformer_patches(model: ModelPatcher) -> dict[str, list[object]]: + """Return the cloned MODEL's mutable transformer patch mapping.""" + + options = model.model_options.get("transformer_options") + if not isinstance(options, dict): + raise TypeError("NegPiP MODEL transformer_options must be a dictionary.") + patches = options.get("patches") + if not isinstance(patches, dict): + raise TypeError("NegPiP MODEL patches must be a dictionary.") + return cast(dict[str, list[object]], patches) + + +def _owned_negpip_callback( + patches: dict[str, list[object]], +) -> tuple[str, str, Any]: + """Resolve exactly one owned family callback from a patch list.""" + + identities = { + ("src.negpip.unet_negpip", "sdxl_attn2_negpip"): ( + "attn2_patch", + "standard", + ), + ("simple_syrup.runtime.negpip.standard", "standard_attn2_negpip"): ( + "attn2_patch", + "standard", + ), + ("src.negpip.anima_negpip", "cosmos_attn2_negpip"): ( + "attn2_patch", + "anima", + ), + ("simple_syrup.runtime.negpip.anima", "anima_attn2_negpip"): ( + "attn2_patch", + "anima", + ), + ("simple_syrup.runtime.negpip.krea2", "krea2_attn1_negpip"): ( + "attn1_patch", + "krea2", + ), + } + matches: list[tuple[str, str, Any]] = [] + observed: list[str] = [] + for patch_name, callbacks in patches.items(): + if not isinstance(callbacks, list): + raise TypeError("NegPiP transformer patches must be callback lists.") + for callback in callbacks: + module = getattr(callback, "__module__", None) + qualname = getattr(callback, "__qualname__", None) + observed.append(f"{patch_name}:{module}.{qualname}") + resolved = None + if isinstance(module, str) and isinstance(qualname, str): + resolved = next( + ( + value + for ( + module_suffix, + expected_qualname, + ), value in identities.items() + if ( + module == module_suffix + or module.endswith(f".{module_suffix}") + ) + and qualname == expected_qualname + ), + None, + ) + if resolved is not None: + matches.append((*resolved, callback)) + if len(matches) != 1: + raise ValueError( + "NegPiP instrumentation requires exactly one owned family callback; " + f"observed {observed!r}." + ) + return matches[0] + + +def _extra_options( + args: tuple[object, ...], + kwargs: dict[str, object], +) -> dict[str, Any]: + """Read Comfy's positional or keyword attention option mapping.""" + + options = kwargs.get("extra_options") + if options is None and args: + options = args[-1] + if not isinstance(options, dict): + return {} + return cast(dict[str, Any], options) + + +def _observe_result( + state: _NegpipProbeState, + *, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + result: object, + options: dict[str, Any], +) -> None: + """Validate one family-specific callback result against its live inputs.""" + + if state.family == "standard": + _observe_standard(state, query, key, value, result) + return + _observe_masked(state, query, key, value, result, options) + + +def _observe_standard( + state: _NegpipProbeState, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + result: object, +) -> None: + """Verify the standard interleaved split and count signed value pairs.""" + + if not isinstance(result, tuple) or len(result) != 3: + raise TypeError("Standard NegPiP must return a Q/K/V tuple.") + output_query, output_key, output_value = result + if output_query is not query: + raise ValueError("Standard NegPiP changed attention queries.") + if not isinstance(output_key, torch.Tensor) or not isinstance( + output_value, torch.Tensor + ): + raise TypeError("Standard NegPiP must return tensor keys and values.") + if not torch.equal(output_key, key[:, 0::2]): + raise ValueError("Standard NegPiP did not select magnitude key positions.") + if not torch.equal(output_value, value[:, 1::2]): + raise ValueError("Standard NegPiP did not select signed value positions.") + pair_delta = value[:, 0::2] - value[:, 1::2] + signed_pairs = torch.any(pair_delta != 0, dim=-1) + negative_locations = signed_pairs.nonzero().tolist() + negative_positions = sorted({int(location[1]) for location in negative_locations}) + _record_observation( + state, + value, + output_value, + negative=bool(negative_locations), + negative_positions=negative_positions, + negative_locations=negative_locations, + ) + + +def _observe_masked( + state: _NegpipProbeState, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + result: object, + options: dict[str, Any], +) -> None: + """Verify Anima or Krea applies a binary sign mask only to values.""" + + if not isinstance(result, dict): + raise TypeError("Masked NegPiP must return an attention tensor dictionary.") + if result.get("q") is not query or result.get("k") is not key: + raise ValueError("Masked NegPiP changed attention queries or keys.") + output_value = result.get("v") + if not isinstance(output_value, torch.Tensor): + raise TypeError("Masked NegPiP must return tensor values.") + mask_key = ANIMA_MASK_KEY if state.family == "anima" else KREA_MASK_KEY + multiplier = options.get(mask_key) + if not isinstance(multiplier, torch.Tensor): + raise TypeError("Masked NegPiP callback did not receive its sign tensor.") + negative_mask = multiplier[:, :, 0] < 0 + negative_locations = negative_mask.nonzero().tolist() + negative = bool(negative_locations) + negative_positions = sorted({int(location[1]) for location in negative_locations}) + state.mask_shape = list(multiplier.shape) + if state.family == "anima": + expected = value * multiplier + else: + image_slice = options.get("img_slice") + if not isinstance(image_slice, (list, tuple)) or len(image_slice) != 2: + raise ValueError("Krea NegPiP did not receive its text/image boundary.") + text_length = image_slice[0] + if not isinstance(text_length, int): + raise TypeError("Krea NegPiP text boundary must be an integer.") + state.text_length = text_length + expected = value.clone() + expected[:, :, :text_length] *= multiplier.to(value).unsqueeze(1) + if not torch.equal(output_value[:, :, text_length:], value[:, :, text_length:]): + raise ValueError("Krea NegPiP changed image or reference values.") + if not torch.equal(output_value, expected): + raise ValueError("Masked NegPiP values do not match the live sign tensor.") + _record_observation( + state, + value, + output_value, + negative=negative, + negative_positions=negative_positions, + negative_locations=negative_locations, + ) + + +def _record_observation( + state: _NegpipProbeState, + source: torch.Tensor, + output: torch.Tensor, + *, + negative: bool, + negative_positions: list[int], + negative_locations: list[list[int]], +) -> None: + """Record one proven callback execution under the process-local lock.""" + + with _STATE_LOCK: + state.attention_calls += 1 + state.negative_mask_calls += int(negative) + if len(negative_positions) > state.negative_token_count: + state.negative_token_count = len(negative_positions) + state.negative_token_positions = negative_positions + state.negative_token_locations = negative_locations + if state.input_value_shape is None: + state.input_value_shape = list(source.shape) + state.output_value_shape = list(output.shape) diff --git a/tools/negpip_integration/__init__.py b/tools/negpip_integration/__init__.py new file mode 100644 index 0000000..62a0544 --- /dev/null +++ b/tools/negpip_integration/__init__.py @@ -0,0 +1,5 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Build and execute isolated automatic NegPiP integration proofs.""" diff --git a/tools/negpip_integration/run.py b/tools/negpip_integration/run.py new file mode 100644 index 0000000..aa06b12 --- /dev/null +++ b/tools/negpip_integration/run.py @@ -0,0 +1,429 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Run isolated live sampling proof for automatic NegPiP on every family.""" + +from __future__ import annotations + +import json +import os +from contextlib import ExitStack +from pathlib import Path +from typing import cast + +from tools.comfy_api import JsonObject +from tools.comfy_integration.artifacts import IntegrationArtifacts +from tools.comfy_integration.default_paths import default_comfy_root +from tools.comfy_integration.history_output import extract_saved_image +from tools.comfy_integration.loopback_port import is_loopback_port_available +from tools.comfy_integration.managed_model_links import ( + ManagedComfyModelLinks, + ManagedModelLink, +) +from tools.comfy_integration.managed_server import ManagedComfyServer + +from .upstream_parity import prove_upstream_parity +from .visual_proof import NegpipVisualProofRecorder +from .workflow import ( + BuiltNegpipLiveWorkflow, + NegpipFixtureSelections, + NegpipLiveFamily, + NegpipLiveWorkflowBuilder, +) + +MODEL_LIBRARY_ENVIRONMENT_VARIABLE = "SIMPLE_SYRUP_MODEL_LIBRARY" +REFINER_FIXTURE_ENVIRONMENT_VARIABLE = "SIMPLE_SYRUP_SDXL_REFINER_FIXTURE" +COMFY_ROOT = default_comfy_root() +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +PPM_ROOT = REPOSITORY_ROOT / ".codex" / "references" / "ComfyUI-ppm-6c6c3601" +ARTIFACT_ROOT = COMFY_ROOT / "benchmark_artifacts" / "negpip-automatic" +SELECTIONS = NegpipFixtureSelections( + sd1_checkpoint=r"simple_syrup_negpip\sd1.safetensors", + sdxl_checkpoint=r"simple_syrup_negpip\sdxl.safetensors", + sdxl_refiner_checkpoint=r"simple_syrup_negpip\sdxl-refiner.safetensors", + anima_diffusion=r"simple_syrup_negpip\anima.safetensors", + anima_text_encoder=r"simple_syrup_negpip\anima-text-encoder.safetensors", + krea2_diffusion=r"simple_syrup_negpip\krea2.safetensors", + krea2_text_encoder=r"simple_syrup_negpip\krea2-text-encoder.safetensors", + qwen_image_vae=r"simple_syrup_negpip\qwen-image-vae.safetensors", +) +PPM_BASELINE_FAMILIES = frozenset( + { + NegpipLiveFamily.SD1, + NegpipLiveFamily.SDXL, + NegpipLiveFamily.SDXL_REFINER, + NegpipLiveFamily.ANIMA, + } +) + + +def execute() -> Path: + """Run controls and one sampled negative-weight case per model family.""" + + artifacts = IntegrationArtifacts(ARTIFACT_ROOT) + model_library = _model_library_root() + refiner_fixture = _refiner_fixture_path() + builder = NegpipLiveWorkflowBuilder(SELECTIONS) + visual_recorder = NegpipVisualProofRecorder(artifacts.root) + workflows = tuple( + workflow + for family in NegpipLiveFamily + for workflow in _family_workflows(builder, artifacts.run_id, family) + ) + required = frozenset().union( + *(workflow.required_node_ids for workflow in workflows) + ) + links = ManagedComfyModelLinks( + model_root=COMFY_ROOT / "models", + links=_model_links(model_library, refiner_fixture), + ) + parity = prove_upstream_parity(PPM_ROOT) + result: dict[str, object] = { + "run_id": artifacts.run_id, + "upstream_parity": parity, + "families": {}, + } + port: int | None = None + try: + with ExitStack() as stack: + stack.enter_context(links) + running = stack.enter_context( + ManagedComfyServer( + comfy_root=COMFY_ROOT, + artifacts=artifacts, + required_node_ids=required, + readiness_timeout=300.0, + launch_arguments=( + "--disable-all-custom-nodes", + "--whitelist-custom-nodes", + "SimpleSyrup", + "SimpleSyrupBenchmarkProbe", + "comfyui-prompt-control", + "comfyui-ppm", + ), + ) + ) + port = running.port + result["port"] = port + result["isolated_custom_nodes"] = [ + "SimpleSyrup", + "SimpleSyrupBenchmarkProbe", + "comfyui-prompt-control", + "comfyui-ppm", + ] + family_results = cast(dict[str, object], result["families"]) + for workflow in workflows: + prompt_id = running.client.submit(workflow.prompt) + history = running.client.wait_for_history(prompt_id, timeout=1800.0) + evidence = _parse_history(history, workflow) + evidence["prompt_id"] = prompt_id + mode = workflow.mode + reference = extract_saved_image(history, workflow.image_node_id) + evidence["image"] = visual_recorder.record( + workflow.family, + mode, + running.client.download_image(reference), + ) + family = cast( + dict[str, object], + family_results.setdefault( + workflow.family.value, + {}, + ), + ) + family[mode] = evidence + ( + artifacts.root / f"{workflow.family.value}-{mode}.history.json" + ).write_text( + json.dumps(history, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + if not links.cleaned: + raise RuntimeError("Managed model aliases were not cleaned.") + if port is None or not is_loopback_port_available(port): + raise RuntimeError("Managed Comfy custom port was not released.") + result["visual_proof"] = visual_recorder.finalize( + cast(JsonObject, result["families"]) + ) + _validate_complete_result(result) + proof_path = artifacts.root / "negpip-proof.json" + proof_path.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + artifacts.record_cleanup(process_running=False, port_available=True) + return proof_path + except BaseException as error: + artifacts.record_failure(error) + raise + + +def _model_links( + model_library: Path, + refiner_fixture: Path, +) -> tuple[ManagedModelLink, ...]: + """Return exact external fixtures and temporary Comfy selection aliases.""" + + return ( + ManagedModelLink( + model_library + / "checkpoints" + / "SD 1.5" + / "abyssorangemix3AOM3_aom3a3.safetensors", + "checkpoints", + SELECTIONS.sd1_checkpoint, + ), + ManagedModelLink( + model_library + / "checkpoints" + / "SDXL" + / "juggernautXL_juggXIByRundiffusion.safetensors", + "checkpoints", + SELECTIONS.sdxl_checkpoint, + ), + ManagedModelLink( + refiner_fixture, + "checkpoints", + SELECTIONS.sdxl_refiner_checkpoint, + ), + ManagedModelLink( + model_library / "diffusion_models" / "Anima" / "anima_baseV10.safetensors", + "diffusion_models", + SELECTIONS.anima_diffusion, + ), + ManagedModelLink( + model_library / "text_encoders" / "qwen" / "qwen_3_06b_base.safetensors", + "text_encoders", + SELECTIONS.anima_text_encoder, + ), + ManagedModelLink( + model_library + / "diffusion_models" + / "Krea2" + / "redcraftHybridH3Krea2dual_11INT8INT4_fp8.safetensors", + "diffusion_models", + SELECTIONS.krea2_diffusion, + ), + ManagedModelLink( + model_library + / "text_encoders" + / "qwen" + / "qwen3vl_4b_fp8_scaled.safetensors", + "text_encoders", + SELECTIONS.krea2_text_encoder, + ), + ManagedModelLink( + model_library / "VAE" / "qwen" / "qwen_image_vae.safetensors", + "vae", + SELECTIONS.qwen_image_vae, + ), + ) + + +def _family_workflows( + builder: NegpipLiveWorkflowBuilder, + run_id: str, + family: NegpipLiveFamily, +) -> tuple[BuiltNegpipLiveWorkflow, ...]: + """Keep each control, automatic, and PPM oracle execution adjacent.""" + + workflows = [ + builder.build( + family, + run_id=f"{run_id}:{family.value}:control", + trigger=False, + ), + builder.build( + family, + run_id=f"{run_id}:{family.value}:negative", + trigger=True, + ), + ] + if family in PPM_BASELINE_FAMILIES: + workflows.append( + builder.build( + family, + run_id=f"{run_id}:{family.value}:ppm-baseline", + trigger=True, + baseline_ppm=True, + ) + ) + return tuple(workflows) + + +def _model_library_root() -> Path: + """Return the explicit absolute external fixture library root.""" + + configured = os.environ.get(MODEL_LIBRARY_ENVIRONMENT_VARIABLE) + if not configured: + raise RuntimeError( + f"Set {MODEL_LIBRARY_ENVIRONMENT_VARIABLE} to the model fixture root." + ) + root = Path(configured).expanduser() + if not root.is_absolute(): + raise ValueError( + f"{MODEL_LIBRARY_ENVIRONMENT_VARIABLE} must be an absolute path." + ) + if not root.is_dir(): + raise FileNotFoundError("Configured model fixture library does not exist.") + return root.resolve() + + +def _refiner_fixture_path() -> Path: + """Return the explicit SDXL Refiner checkpoint used by live proof.""" + + configured = os.environ.get(REFINER_FIXTURE_ENVIRONMENT_VARIABLE) + if not configured: + raise RuntimeError( + f"Set {REFINER_FIXTURE_ENVIRONMENT_VARIABLE} to a refiner checkpoint." + ) + fixture = Path(configured).expanduser() + if not fixture.is_absolute(): + raise ValueError( + f"{REFINER_FIXTURE_ENVIRONMENT_VARIABLE} must be an absolute path." + ) + if not fixture.is_file(): + raise FileNotFoundError("Configured SDXL Refiner fixture does not exist.") + return fixture.resolve() + + +def _parse_history( + history: JsonObject, + workflow: BuiltNegpipLiveWorkflow, +) -> JsonObject: + """Extract exact control or sampled runtime evidence from completed history.""" + + status = _mapping(history.get("status"), "history.status") + if status.get("status_str") != "success" or status.get("completed") is not True: + raise RuntimeError( + f"Managed NegPiP workflow failed: {status.get('messages')!r}" + ) + outputs = _mapping(history.get("outputs"), "history.outputs") + modifier = _single_output( + outputs, + workflow.modifier_node_id, + "model_modifier_snapshot", + ) + result: JsonObject = {"modifier": modifier} + if workflow.runtime_node_id is None: + return result + if workflow.conditioning_node_id is None: + raise ValueError("Triggered NegPiP workflow is missing conditioning evidence.") + result["runtime"] = _single_output( + outputs, + workflow.runtime_node_id, + "negpip_runtime_evidence", + ) + result["conditioning"] = _single_output( + outputs, + workflow.conditioning_node_id, + "conditioning_batch_snapshot", + ) + return result + + +def _single_output( + outputs: JsonObject, + node_id: str, + field: str, +) -> JsonObject: + """Return one exact UI evidence record.""" + + node = _mapping(outputs.get(node_id), f"outputs[{node_id}]") + values = node.get(field) + if not isinstance(values, list) or len(values) != 1: + raise ValueError(f"{field} must contain exactly one record.") + return _mapping(values[0], field) + + +def _mapping(value: object, field: str) -> JsonObject: + """Narrow one JSON object with string keys.""" + + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise TypeError(f"{field} must be a JSON object.") + return cast(JsonObject, value) + + +def _validate_complete_result(result: dict[str, object]) -> None: + """Fail unless gating and live negative execution passed for every family.""" + + families = cast(dict[str, object], result["families"]) + if set(families) != {family.value for family in NegpipLiveFamily}: + raise ValueError("Managed NegPiP proof did not cover every model family.") + for family_name, family_value in families.items(): + family = cast(dict[str, object], family_value) + control = cast(dict[str, object], family["control"]) + negative = cast(dict[str, object], family["negative"]) + control_modifier = cast(dict[str, object], control["modifier"]) + negative_modifier = cast(dict[str, object], negative["modifier"]) + runtime = cast(dict[str, object], negative["runtime"]) + control_image = cast(dict[str, object], control["image"]) + negative_image = cast(dict[str, object], negative["image"]) + comparison = cast(dict[str, object], family["image_comparison"]) + if control_modifier.get("ppm_negpip") is not False: + raise ValueError(f"{family_name} control unexpectedly enabled NegPiP.") + if negative_modifier.get("ppm_negpip") is not True: + raise ValueError(f"{family_name} negative prompt did not enable NegPiP.") + if ( + not isinstance(runtime.get("attention_calls"), int) + or cast(int, runtime["attention_calls"]) < 1 + ): + raise ValueError(f"{family_name} did not execute NegPiP attention.") + if ( + not isinstance(runtime.get("negative_mask_calls"), int) + or cast(int, runtime["negative_mask_calls"]) < 1 + ): + raise ValueError(f"{family_name} never applied a negative token sign.") + if runtime.get("invariant_failures") != []: + raise ValueError(f"{family_name} reported live NegPiP invariant failures.") + if control_image.get("rgb_sha256") == negative_image.get("rgb_sha256"): + raise ValueError(f"{family_name} decoded image pair is identical.") + changed_pixels = comparison.get("changed_pixels") + if not isinstance(changed_pixels, int) or changed_pixels < 1: + raise ValueError(f"{family_name} has no visible pixel differences.") + if family_name in {item.value for item in PPM_BASELINE_FAMILIES}: + baseline = cast(dict[str, object], family["ppm_baseline"]) + baseline_conditioning = cast(dict[str, object], baseline["conditioning"]) + negative_conditioning = cast(dict[str, object], negative["conditioning"]) + parity = cast(dict[str, object], family["ppm_automatic_comparison"]) + mean_delta = parity.get("mean_absolute_rgb_delta") + p99_delta = parity.get("p99_channel_delta") + if ( + not isinstance(mean_delta, (int, float)) + or mean_delta > 0.5 + or not isinstance(p99_delta, (int, float)) + or p99_delta > 5.0 + ): + raise ValueError( + f"{family_name} automatic image exceeded pinned PPM numerical " + "parity tolerance." + ) + for side in ("positive", "negative"): + if baseline_conditioning.get(side) != negative_conditioning.get(side): + raise ValueError( + f"{family_name} {side} conditioning diverged from pinned PPM." + ) + baseline_runtime = cast(dict[str, object], baseline["runtime"]) + for field in ( + "family", + "patch_name", + "attention_calls", + "negative_mask_calls", + "input_value_shape", + "output_value_shape", + "mask_shape", + "text_length", + "negative_token_count", + "negative_token_positions", + "negative_token_locations", + "invariant_failures", + ): + if baseline_runtime.get(field) != runtime.get(field): + raise ValueError( + f"{family_name} live {field} diverged from pinned PPM." + ) + + +if __name__ == "__main__": + print(execute()) diff --git a/tools/negpip_integration/upstream_parity.py b/tools/negpip_integration/upstream_parity.py new file mode 100644 index 0000000..37af3d1 --- /dev/null +++ b/tools/negpip_integration/upstream_parity.py @@ -0,0 +1,147 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Compare owned SD/Anima behavior to the pinned local PPM baseline.""" + +from __future__ import annotations + +import importlib.util +import subprocess +from pathlib import Path +from types import ModuleType +from typing import Any, cast + +import torch + +from simple_syrup.runtime.negpip.anima import ( + CONDITION_MASK_KEY, + anima_attn2_negpip, + anima_extra_conds_negpip_wrapper, +) +from simple_syrup.runtime.negpip.standard import encode_token_weights_negpip + +PPM_REVISION = "6c6c360155cace9d7091306c1b8e26d9c7438620" + + +class _Encoder: + """Produce deterministic embeddings for exact upstream parity.""" + + special_tokens: dict[str, int] = {} + + def gen_empty_tokens( + self, + special_tokens: dict[str, int], + length: int, + ) -> list[int]: + """Return one equal-length empty prompt.""" + + del special_tokens + return [0] * length + + def encode(self, sections: list[list[object]]) -> tuple[torch.Tensor, None]: + """Map each integer token to one two-channel embedding.""" + + rows = [ + [[float(cast(int, token)), float(cast(int, token)) + 0.25] for token in row] + for row in sections + ] + return torch.tensor(rows), None + + +def prove_upstream_parity(ppm_root: Path) -> dict[str, object]: + """Return exact revision and tensor equality evidence against local PPM.""" + + resolved = ppm_root.resolve() + revision = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=resolved, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if revision != PPM_REVISION: + raise ValueError( + f"PPM baseline revision is {revision}, expected {PPM_REVISION}." + ) + upstream_standard = _load_module( + "simple_syrup_ppm_unet_negpip", + resolved / "src" / "negpip" / "unet_negpip.py", + ) + upstream_anima = _load_module( + "simple_syrup_ppm_anima_negpip", + resolved / "src" / "negpip" / "anima_negpip.py", + ) + encoder = cast(Any, _Encoder()) + token_pairs: list[list[tuple[object, float]]] = [[(3, -2.0), (5, 0.5), (7, 1.0)]] + owned_standard = encode_token_weights_negpip(encoder, token_pairs) + baseline_standard = upstream_standard.encode_token_weights_negpip( + encoder, + token_pairs, + ) + standard_equal = _tuple_tensors_equal(owned_standard, baseline_standard) + + def base_extra(**kwargs: object) -> dict[str, object]: + return {"weights": kwargs["t5xxl_weights"]} + + weights = torch.tensor([-2.0, 0.5, 1.0]) + owned_extra = anima_extra_conds_negpip_wrapper(base_extra)( + t5xxl_weights=weights.clone() + ) + baseline_extra = upstream_anima.anima_extra_conds_negpip_wrapper(base_extra)( + t5xxl_weights=weights.clone() + ) + owned_mask = cast(Any, owned_extra[CONDITION_MASK_KEY]).cond + baseline_mask = cast(Any, baseline_extra[CONDITION_MASK_KEY]).cond + query = torch.ones((1, 1, 512, 2)) + key = query * 2 + value = query * 3 + owned_attention = anima_attn2_negpip( + query, + key, + value, + extra_options={"ppm_negpip_mask": owned_mask}, + ) + baseline_attention = upstream_anima.cosmos_attn2_negpip( + query, + key, + value, + extra_options={"ppm_negpip_mask": baseline_mask}, + ) + anima_equal = torch.equal(owned_mask, baseline_mask) and torch.equal( + cast(torch.Tensor, owned_attention["v"]), + cast(torch.Tensor, baseline_attention["v"]), + ) + if not standard_equal or not anima_equal: + raise ValueError("Owned NegPiP behavior diverged from the pinned PPM baseline.") + return { + "ppm_revision": revision, + "standard_encoding_equal": standard_equal, + "anima_mask_and_attention_equal": anima_equal, + "standard_output_shape": list(cast(torch.Tensor, owned_standard[0]).shape), + "anima_mask_shape": list(owned_mask.shape), + } + + +def _load_module(name: str, path: Path) -> ModuleType: + """Load one exact baseline file without registering its Comfy nodes.""" + + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load PPM source file: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _tuple_tensors_equal(left: tuple[object, ...], right: tuple[object, ...]) -> bool: + """Compare the deterministic tensor/None result used by the parity fixture.""" + + if len(left) != len(right): + return False + return all( + torch.equal(left_item, right_item) + if isinstance(left_item, torch.Tensor) and isinstance(right_item, torch.Tensor) + else left_item == right_item + for left_item, right_item in zip(left, right, strict=True) + ) diff --git a/tools/negpip_integration/visual_proof.py b/tools/negpip_integration/visual_proof.py new file mode 100644 index 0000000..1b5ac6e --- /dev/null +++ b/tools/negpip_integration/visual_proof.py @@ -0,0 +1,292 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Persist decoded NegPiP images and assemble visible paired proof.""" + +from __future__ import annotations + +import hashlib +import io +import math +from pathlib import Path +from typing import cast + +from PIL import Image, ImageChops, ImageDraw, ImageStat + +from tools.comfy_api import JsonObject +from tools.comfy_integration.portable_font import load_label_font + +from .workflow import NEGATIVE_WEIGHT_LABEL, NegpipLiveFamily + +EXPECTED_IMAGE_SIZE = (512, 512) + + +class NegpipVisualProofRecorder: + """Own original decoded images, pixel comparisons, and a contact sheet.""" + + def __init__(self, root: Path) -> None: + """Retain one existing managed artifact directory.""" + + self._root = root.resolve() + + def record( + self, + family: NegpipLiveFamily, + mode: str, + image_bytes: bytes, + ) -> JsonObject: + """Validate and persist one original Comfy PNG.""" + + if mode not in {"control", "negative", "ppm_baseline"}: + raise ValueError(f"Unknown NegPiP visual proof mode: {mode!r}.") + if not image_bytes: + raise ValueError("NegPiP visual proof image is empty.") + with Image.open(io.BytesIO(image_bytes)) as decoded: + decoded.load() + if decoded.format != "PNG": + raise ValueError("NegPiP visual proof output must be a PNG.") + image = decoded.convert("RGB") + if image.size != EXPECTED_IMAGE_SIZE: + raise ValueError( + "NegPiP visual proof image must be 512x512; " + f"received {image.size[0]}x{image.size[1]}." + ) + path = self._root / f"{family.value}-{mode}.png" + path.write_bytes(image_bytes) + pixels = image.tobytes() + return { + "file": path.name, + "png_sha256": hashlib.sha256(image_bytes).hexdigest(), + "rgb_sha256": hashlib.sha256(pixels).hexdigest(), + "size_bytes": len(image_bytes), + "width": image.width, + "height": image.height, + "rgb_dynamic_range": max(pixels) - min(pixels), + } + + def finalize(self, families: JsonObject) -> JsonObject: + """Require every pair, measure its pixels, and create a labeled sheet.""" + + expected = {family.value for family in NegpipLiveFamily} + if set(families) != expected: + raise ValueError("NegPiP visible proof does not cover every model family.") + for family in NegpipLiveFamily: + family_result = _mapping(families.get(family.value), family.value) + control = _mapping(family_result.get("control"), "control") + negative = _mapping(family_result.get("negative"), "negative") + comparison = self._compare( + _image_path(self._root, control), + _image_path(self._root, negative), + ) + if comparison["changed_pixels"] == 0: + raise ValueError(f"{family.value} control and NegPiP images are equal.") + family_result["image_comparison"] = comparison + ppm_baseline = family_result.get("ppm_baseline") + if ppm_baseline is not None: + baseline = _mapping(ppm_baseline, "ppm_baseline") + family_result["ppm_automatic_comparison"] = self._compare( + _image_path(self._root, baseline), + _image_path(self._root, negative), + ) + sheet = self._contact_sheet(families) + data = sheet.read_bytes() + with Image.open(io.BytesIO(data)) as decoded: + width, height = decoded.size + return { + "file": sheet.name, + "png_sha256": hashlib.sha256(data).hexdigest(), + "size_bytes": len(data), + "width": width, + "height": height, + } + + @staticmethod + def _compare(control_path: Path, negative_path: Path) -> JsonObject: + """Return exact RGB difference evidence for a matched-seed pair.""" + + with Image.open(control_path) as control_source: + control = control_source.convert("RGB") + with Image.open(negative_path) as negative_source: + negative = negative_source.convert("RGB") + if control.size != negative.size: + raise ValueError("NegPiP visual proof pair dimensions do not match.") + difference = ImageChops.difference(control, negative) + difference_bytes = difference.tobytes() + element_count = len(difference_bytes) + changed_pixels = sum( + any(difference_bytes[index : index + 3]) + for index in range(0, element_count, 3) + ) + absolute_sum = sum(difference_bytes) + squared_sum = sum(value * value for value in difference_bytes) + ordered_deltas = sorted(difference_bytes) + p99_index = math.ceil(len(ordered_deltas) * 0.99) - 1 + stat = ImageStat.Stat(difference) + return { + "changed_pixels": changed_pixels, + "total_pixels": control.width * control.height, + "mean_absolute_rgb_delta": absolute_sum / element_count, + "root_mean_square_rgb_delta": math.sqrt(squared_sum / element_count), + "channel_mean_absolute_delta": list(stat.mean), + "maximum_channel_delta": max(difference_bytes, default=0), + "p99_channel_delta": ordered_deltas[p99_index], + "difference_bbox": list(difference.getbbox() or (0, 0, 0, 0)), + } + + def _contact_sheet(self, families: JsonObject) -> Path: + """Render original images and runtime observations into one proof sheet.""" + + margin = 28 + label_width = 230 + image_size = 340 + column_gap = 20 + header_height = 150 + row_height = 470 + width = margin * 2 + label_width + image_size * 3 + column_gap * 2 + height = header_height + row_height * len(NegpipLiveFamily) + margin + canvas = Image.new("RGB", (width, height), (18, 20, 24)) + draw = ImageDraw.Draw(canvas) + title_font = load_label_font(30) + heading_font = load_label_font(22) + detail_font = load_label_font(16) + draw.text( + (margin, 20), + "Automatic NegPiP - decoded ComfyUI proof", + fill="white", + font=title_font, + ) + draw.text( + (margin, 62), + (f"Same seed 4,205,191 | 512x512 | target: {NEGATIVE_WEIGHT_LABEL}"), + fill=(198, 204, 214), + font=detail_font, + ) + control_x = margin + label_width + baseline_x = control_x + image_size + column_gap + negative_x = baseline_x + image_size + column_gap + draw.text( + (control_x, 102), + "CONTROL (+1)", + fill=(120, 205, 255), + font=heading_font, + ) + draw.text( + (baseline_x, 102), + "PINNED PPM (-1)", + fill=(255, 170, 120), + font=heading_font, + ) + draw.text( + (negative_x, 102), + "SIMPLESYRUP AUTO (-1)", + fill=(170, 235, 165), + font=heading_font, + ) + for row, family in enumerate(NegpipLiveFamily): + top = header_height + row * row_height + family_result = _mapping(families.get(family.value), family.value) + control = _mapping(family_result.get("control"), "control") + negative = _mapping(family_result.get("negative"), "negative") + runtime = _mapping(negative.get("runtime"), "runtime") + comparison = _mapping( + family_result.get("image_comparison"), "image_comparison" + ) + with Image.open(_image_path(self._root, control)) as source: + control_image = source.convert("RGB").resize( + (image_size, image_size), Image.Resampling.LANCZOS + ) + with Image.open(_image_path(self._root, negative)) as source: + negative_image = source.convert("RGB").resize( + (image_size, image_size), Image.Resampling.LANCZOS + ) + canvas.paste(control_image, (control_x, top)) + ppm_baseline = family_result.get("ppm_baseline") + if ppm_baseline is None: + draw.rectangle( + ( + baseline_x, + top, + baseline_x + image_size, + top + image_size, + ), + fill=(30, 33, 39), + outline=(75, 80, 90), + width=2, + ) + draw.multiline_text( + (baseline_x + 36, top + 130), + "PPM has no Krea 2 path\n\nSimpleSyrup extension test ->", + fill=(180, 186, 196), + font=detail_font, + spacing=8, + ) + else: + baseline = _mapping(ppm_baseline, "ppm_baseline") + with Image.open(_image_path(self._root, baseline)) as source: + baseline_image = source.convert("RGB").resize( + (image_size, image_size), Image.Resampling.LANCZOS + ) + canvas.paste(baseline_image, (baseline_x, top)) + canvas.paste(negative_image, (negative_x, top)) + parity_detail = f"concept: {NEGATIVE_WEIGHT_LABEL}" + if ppm_baseline is not None: + parity = _mapping( + family_result.get("ppm_automatic_comparison"), + "ppm comparison", + ) + mean_delta = cast(float, parity.get("mean_absolute_rgb_delta")) + parity_detail = f"PPM~AUTO MAE: {float(mean_delta):.3f}/255" + draw.text( + (margin, top + 8), + family.value.upper().replace("_", " "), + fill="white", + font=heading_font, + ) + draw.multiline_text( + (margin, top + 52), + ( + "control marker: OFF\n" + "trigger marker: ON\n" + f"callback: {runtime.get('family')}\n" + f"calls: {runtime.get('attention_calls')}\n" + f"negative signs: {runtime.get('negative_mask_calls')}\n" + f"negative tokens: {runtime.get('negative_token_count')}\n" + f"control changed: {comparison.get('changed_pixels')}\n" + f"{parity_detail}" + ), + fill=(200, 206, 216), + font=detail_font, + spacing=8, + ) + draw.line( + (margin, top + row_height - 18, width - margin, top + row_height - 18), + fill=(64, 68, 76), + width=2, + ) + path = self._root / "negpip-visible-proof.png" + canvas.save(path, format="PNG") + return path + + +def _image_path(root: Path, case: JsonObject) -> Path: + """Resolve one recorded image inside the artifact root.""" + + image = _mapping(case.get("image"), "image") + relative = image.get("file") + if not isinstance(relative, str) or not relative: + raise ValueError("NegPiP visual proof image file is missing.") + path = (root / relative).resolve() + if path.parent != root: + raise ValueError("NegPiP visual proof image escaped its artifact root.") + if not path.is_file(): + raise FileNotFoundError(f"NegPiP visual proof image is missing: {path}.") + return path + + +def _mapping(value: object, field: str) -> JsonObject: + """Narrow one JSON object with string keys.""" + + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise TypeError(f"{field} must be a JSON object.") + return cast(JsonObject, value) diff --git a/tools/negpip_integration/workflow.py b/tools/negpip_integration/workflow.py new file mode 100644 index 0000000..893c58e --- /dev/null +++ b/tools/negpip_integration/workflow.py @@ -0,0 +1,327 @@ +# SimpleSyrup - workflow-focused ComfyUI extensions for image generation +# Copyright (C) 2026 Artificial Sweetener and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Build focused loader-to-sampler automatic NegPiP workflows.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + +from tools.anima_workflow_graph import AnimaWorkflowGraph +from tools.comfy_api import JsonObject + +CONTROL_PROMPT = ( + "studio portrait of a person wearing a bright red jacket, plain gray background" +) +NEGATIVE_WEIGHT_LABEL = "bright (red:-1.0) jacket" +NEGATIVE_WEIGHT_PROMPT = ( + f"studio portrait of a person wearing a {NEGATIVE_WEIGHT_LABEL}, " + "plain gray background" +) +REFINER_BASE_PROMPT = ( + "studio portrait of a person wearing a jacket, plain gray background" +) +REFINER_SWITCH_STEP = 16 + + +class NegpipLiveFamily(StrEnum): + """Identify every materially distinct supported live model path.""" + + SD1 = "sd1" + SDXL = "sdxl" + SDXL_REFINER = "sdxl_refiner" + ANIMA = "anima" + KREA2 = "krea2" + + +@dataclass(frozen=True, slots=True) +class NegpipFixtureSelections: + """Name exact model selections exposed to the isolated Comfy server.""" + + sd1_checkpoint: str + sdxl_checkpoint: str + sdxl_refiner_checkpoint: str + anima_diffusion: str + anima_text_encoder: str + krea2_diffusion: str + krea2_text_encoder: str + qwen_image_vae: str + + +@dataclass(frozen=True, slots=True) +class BuiltNegpipLiveWorkflow: + """Retain one graph and its exact evidence node identities.""" + + prompt: dict[str, JsonObject] + family: NegpipLiveFamily + run_id: str + runtime_node_id: str | None + modifier_node_id: str + conditioning_node_id: str | None + image_node_id: str + mode: str + + @property + def required_node_ids(self) -> frozenset[str]: + """Return every Comfy node contract named by this workflow.""" + + return frozenset(str(node["class_type"]) for node in self.prompt.values()) + + +class NegpipLiveWorkflowBuilder: + """Create triggered sampling and untriggered gate-control workflows.""" + + def __init__(self, selections: NegpipFixtureSelections) -> None: + """Retain exact managed model aliases.""" + + self._selections = selections + + def build( + self, + family: NegpipLiveFamily, + *, + run_id: str, + trigger: bool, + baseline_ppm: bool = False, + ) -> BuiltNegpipLiveWorkflow: + """Build one public Schedule & Encode graph with live evidence.""" + + if baseline_ppm and not trigger: + raise ValueError("The PPM baseline workflow requires a negative weight.") + + graph = AnimaWorkflowGraph() + model, clip, vae = self._loader(graph, family) + if baseline_ppm: + baseline = graph.add("CLIPNegPip", model=model, clip=clip) + model = [baseline, 0] + clip = [baseline, 1] + prompt = NEGATIVE_WEIGHT_PROMPT if trigger else CONTROL_PROMPT + scheduled = graph.add( + "SimpleSyrup.ScheduleAndEncodePromptsWithPromptControl", + model=model, + clip=clip, + positive_prompt=prompt, + negative_prompt="blurry, low quality", + ) + modifier = graph.add( + "SimpleSyrupBenchmark.SnapshotModelModifier", + model=[scheduled, 0], + run_id=f"{run_id}:modifier", + ) + conditioning: str | None = None + sampling_model: list[str | int] = [modifier, 0] + if trigger: + conditioning = graph.add( + "SimpleSyrupBenchmark.SnapshotConditioningBatch", + positive=[scheduled, 1], + negative=[scheduled, 2], + run_id=f"{run_id}:conditioning", + ) + instrumented = graph.add( + "SimpleSyrupBenchmark.InstrumentNegpipModel", + model=[modifier, 0], + run_id=run_id, + ) + sampling_model = [instrumented, 0] + if family is NegpipLiveFamily.SDXL_REFINER: + latent, vae = self._sdxl_base_stage(graph) + else: + latent = self._latent(graph, family) + steps, cfg, sampler_name, scheduler = self._sampling(family) + if family is NegpipLiveFamily.SDXL_REFINER: + sampled = graph.add( + "KSamplerAdvanced", + model=sampling_model, + add_noise="disable", + noise_seed=4_205_191, + steps=24, + cfg=cfg, + sampler_name=sampler_name, + scheduler=scheduler, + positive=[scheduled, 1], + negative=[scheduled, 2], + latent_image=latent, + start_at_step=REFINER_SWITCH_STEP, + end_at_step=24, + return_with_leftover_noise="disable", + ) + else: + sampled = graph.add( + "KSampler", + model=sampling_model, + seed=4_205_191, + steps=steps, + cfg=cfg, + sampler_name=sampler_name, + scheduler=scheduler, + positive=[scheduled, 1], + negative=[scheduled, 2], + latent_image=latent, + denoise=1.0, + ) + runtime: str | None = None + decoded_latent: list[str | int] = [sampled, 0] + if trigger: + runtime = graph.add( + "SimpleSyrupBenchmark.ReadNegpipRuntime", + latent=[sampled, 0], + run_id=run_id, + ) + decoded_latent = [runtime, 0] + decoded = graph.add("VAEDecode", samples=decoded_latent, vae=vae) + saved = graph.add( + "SaveImage", + images=[decoded, 0], + filename_prefix=( + "simple_syrup_negpip_proof/" + + run_id.replace(":", "-").replace("\\", "-") + ), + ) + return BuiltNegpipLiveWorkflow( + graph.prompt, + family, + run_id, + runtime, + modifier, + conditioning, + saved, + ("ppm_baseline" if baseline_ppm else "negative" if trigger else "control"), + ) + + def _loader( + self, + graph: AnimaWorkflowGraph, + family: NegpipLiveFamily, + ) -> tuple[list[str | int], list[str | int], list[str | int]]: + """Add the exact family loader and return MODEL/CLIP/VAE references.""" + + if family in { + NegpipLiveFamily.SD1, + NegpipLiveFamily.SDXL, + NegpipLiveFamily.SDXL_REFINER, + }: + selections = { + NegpipLiveFamily.SD1: self._selections.sd1_checkpoint, + NegpipLiveFamily.SDXL: self._selections.sdxl_checkpoint, + NegpipLiveFamily.SDXL_REFINER: ( + self._selections.sdxl_refiner_checkpoint + ), + } + selection = selections[family] + loader = graph.add("CheckpointLoaderSimple", ckpt_name=selection) + return [loader, 0], [loader, 1], [loader, 2] + if family is NegpipLiveFamily.ANIMA: + loader = graph.add( + "SimpleSyrup.SimpleLoadAnima", + diffusion_model=self._selections.anima_diffusion, + quantization="Original", + diffusion_weight_dtype="default", + text_encoder=self._selections.anima_text_encoder, + text_encoder_device="default", + vae=self._selections.qwen_image_vae, + ) + return [loader, 0], [loader, 1], [loader, 2] + loader = graph.add( + "UNETLoader", + unet_name=self._selections.krea2_diffusion, + weight_dtype="default", + ) + clip_loader = graph.add( + "CLIPLoader", + clip_name=self._selections.krea2_text_encoder, + type="krea2", + device="default", + ) + vae_loader = graph.add("VAELoader", vae_name=self._selections.qwen_image_vae) + return [loader, 0], [clip_loader, 0], [vae_loader, 0] + + @staticmethod + def _latent( + graph: AnimaWorkflowGraph, + family: NegpipLiveFamily, + ) -> list[str | int]: + """Add the family's native smallest practical image latent.""" + + if family is NegpipLiveFamily.ANIMA: + node = graph.add( + "EmptyCosmosLatentVideo", + width=512, + height=512, + length=1, + batch_size=1, + ) + elif family is NegpipLiveFamily.KREA2: + node = graph.add( + "EmptySD3LatentImage", + width=512, + height=512, + batch_size=1, + ) + else: + node = graph.add( + "EmptyLatentImage", + width=512, + height=512, + batch_size=1, + ) + return [node, 0] + + def _sdxl_base_stage( + self, + graph: AnimaWorkflowGraph, + ) -> tuple[list[str | int], list[str | int]]: + """Generate a valid high-noise SDXL latent for the refiner proof stage.""" + + base = graph.add( + "CheckpointLoaderSimple", + ckpt_name=self._selections.sdxl_checkpoint, + ) + positive = graph.add( + "CLIPTextEncode", + clip=[base, 1], + text=REFINER_BASE_PROMPT, + ) + negative = graph.add( + "CLIPTextEncode", + clip=[base, 1], + text="blurry, low quality", + ) + latent = graph.add( + "EmptyLatentImage", + width=512, + height=512, + batch_size=1, + ) + sampled = graph.add( + "KSamplerAdvanced", + model=[base, 0], + add_noise="enable", + noise_seed=4_205_191, + steps=24, + cfg=5.0, + sampler_name="euler", + scheduler="normal", + positive=[positive, 0], + negative=[negative, 0], + latent_image=[latent, 0], + start_at_step=0, + end_at_step=REFINER_SWITCH_STEP, + return_with_leftover_noise="enable", + ) + return [sampled, 0], [base, 2] + + @staticmethod + def _sampling( + family: NegpipLiveFamily, + ) -> tuple[int, float, str, str]: + """Return practical multi-step settings for visible family output.""" + + if family is NegpipLiveFamily.ANIMA: + return 16, 1.0, "er_sde", "simple" + if family is NegpipLiveFamily.KREA2: + return 16, 1.0, "euler", "simple" + if family is NegpipLiveFamily.SD1: + return 20, 7.0, "euler", "normal" + return 20, 5.0, "euler", "normal"