DumiFlex
1224d35010
A nested ref in a derivation resolved to empty whenever the wildcard it points
at was not ALSO picked into the same Context node — no matter what the live
library held, and no matter how the ref was filtered.
Root cause is discovery, not resolution, which is why it looked like a
resolution bug. The catalog is synthesised from wildcard modules only, and
`walk_transitive_refs` was seeded from those uuids and followed refs found in
wildcard payloads only. A `@{uuid}` is legal on other surfaces too — most
importantly a derivation rule's action value, and the per-instance
`action_value_overrides` the edit modal writes when a user retargets a branch
on the canvas. Neither was ever scanned.
So such a ref's target reached the catalog only by coincidence: when the user
happened to also have that wildcard picked in the node. Removing it — a
reasonable thing to do once a derivation references it by uuid — dropped it
with nothing to put it back, and the ref became `unknown_ref` → empty string.
`ref_seed_uuids` (engine/modules/snapshot.py) now collects refs from every
module's `payload` AND `instance`, and both catalog builders seed the walk
with them. The instance half matters on its own: a canvas-side retarget lives
there and is not part of the library payload the walker would otherwise see.
Both builders are fixed, not just the graph one. `wp_api/preview.py` had the
same seeding, so the SPA preview resolved against a pool the engine would not
use — that is precisely how "the SPA says 1 of 11 options match, the graph
says 0 of 10" arose. Fixing one and not the other would have left the two
disagreeing in the opposite direction.
Embedded-first precedence is unchanged: a picked module still shadows the
library, so drift detection and workflow reproducibility hold. This only adds
targets that were previously absent entirely. An unresolvable uuid still warns
and resolves to empty rather than raising.
398 lines
16 KiB
Python
398 lines
16 KiB
Python
"""Snapshot freezing, content hashing, legacy-shape coercion, and transitive
|
|
ref walking.
|
|
|
|
A *snapshot* is the JSON shape embedded in a ComfyUI workflow's WP_Context
|
|
node when a user picks a library row. The snapshot is decoupled from the
|
|
library: graph execution reads only the snapshot, never the live DB.
|
|
|
|
Backward-compat: pre-SPA fixed_values modules in saved workflows lack the
|
|
``library_*`` and ``instance`` keys. ``coerce_legacy_module`` upgrades them
|
|
in-place to the canonical shape (still detached, ``library_id is None``).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from collections.abc import Callable, Iterable
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Literal, TypedDict
|
|
|
|
from engine._utils import now_iso as _now_iso
|
|
from engine.syntax.tokenize import tokenize_text
|
|
from engine.syntax.types import TokenKind
|
|
|
|
|
|
class _SourceUser(TypedDict):
|
|
kind: Literal["user"]
|
|
|
|
|
|
class _SourceDep(TypedDict):
|
|
kind: Literal["dep"]
|
|
parent_uuids: list[str]
|
|
|
|
|
|
SnapshotSource = _SourceUser | _SourceDep
|
|
|
|
|
|
class SnapshotEntry(TypedDict):
|
|
"""Canonical shape stored in `__wp_catalog__` and embedded in
|
|
WP_Context workflow JSON. Spec §2.4 — pinned, do not rename fields.
|
|
|
|
`payload_hash` covers `payload` only (not name/description/tags) so
|
|
a rename does not flip drift state. `source` records why the entry
|
|
is in the embed: explicit user pick vs. transitive dep of one or
|
|
more picks (multi-parent supported)."""
|
|
snapshot_version: Literal[1]
|
|
uuid: str # 8 hex chars
|
|
type: str # ModuleType — wildcards-only in catalog (spec §2.7)
|
|
name: str # display label
|
|
payload: dict[str, Any]
|
|
payload_hash: str # SHA-256 of payload
|
|
source: SnapshotSource
|
|
|
|
|
|
def _fresh_instance() -> dict[str, Any]:
|
|
"""Default per-snapshot instance overrides. Fresh dict per call.
|
|
|
|
Invariant: every field defaults to ``None``. The frontend
|
|
"modified-state" predicate is ``value != null``, so any non-None
|
|
sentinel (e.g. ``""`` for a string field, ``False`` for a bool)
|
|
would falsely light up the orange dot, force the modal to default
|
|
to the Instance tab, and tag the section as modified — even on a
|
|
pristine snapshot. Engine readers already tolerate ``None`` (see
|
|
``wildcard_handler.py:216`` for the binding fallback chain and
|
|
``pipeline.py:160`` for the internal-flag coercion), so we keep a
|
|
single contract: ``None`` everywhere = "no override, use library
|
|
default".
|
|
"""
|
|
return {
|
|
"variable_binding": None,
|
|
"enabled_options": None,
|
|
"category_filter": None,
|
|
"option_weights": None,
|
|
# SP2a multi-select + the null toggle — part of the pool-shaping
|
|
# override set, so a frozen snapshot carries them too (all None =
|
|
# "no override"). Keeps this dict honest with the "every field" claim
|
|
# above + the instance schema (engine/schemas/instance_schemas.py).
|
|
"exclude_null": None,
|
|
"pick_min": None,
|
|
"pick_max": None,
|
|
"pick_separator": None,
|
|
"pick_independent": None, # SP2c: multi-pick with replacement when true
|
|
# Pick mode — `None`/missing → `random` (legacy behavior:
|
|
# weighted RNG over enabled options). `pinned` short-circuits
|
|
# the RNG and always picks `pinned_option_id`. `subcategory`
|
|
# is currently identical to `random` engine-side; the modal
|
|
# uses it as a UX framing for "manual subset via enabled_options".
|
|
"mode": None,
|
|
"pinned_option_id": None,
|
|
# Lock — when set, derives a stable per-instance RNG seed
|
|
# independent of the chain's `__wp_node_seed__`. Means the
|
|
# wildcard's pick stays the same value across runs even when
|
|
# the user cycles the Context node's seed. `None` = roll with
|
|
# the chain seed (default).
|
|
"locked_seed": None,
|
|
# Internal — when True, every binding this module produces is
|
|
# marked engine-only by `PipelineEngine.run` (via
|
|
# `__wp_internal_flags__`). Downstream modules still see the
|
|
# value; the public socket payload doesn't. ``None`` is coerced
|
|
# to ``False`` at the read site, so the default stays "off".
|
|
"internal": None,
|
|
"disabled_rule_ids": None,
|
|
"disabled_exception_keys": None,
|
|
"disabled_matrix_cells": None,
|
|
}
|
|
|
|
|
|
def payload_hash(payload: dict[str, Any]) -> str:
|
|
"""SHA-256 of canonical JSON. Stable across key order + whitespace."""
|
|
canonical = json.dumps(
|
|
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
)
|
|
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def freeze_snapshot(library_row: dict[str, Any]) -> dict[str, Any]:
|
|
"""Build the JSON shape that gets embedded in a node's modules array."""
|
|
return {
|
|
"library_id": library_row["id"],
|
|
"library_snapshot_at": _now_iso(),
|
|
"library_version_at_snapshot": library_row["version"],
|
|
"type": library_row["type"],
|
|
"name": library_row["name"],
|
|
"category_id": library_row.get("category_id"),
|
|
"payload": library_row["payload"],
|
|
"instance": _fresh_instance(),
|
|
}
|
|
|
|
|
|
def coerce_legacy_module(raw: dict[str, Any]) -> dict[str, Any]:
|
|
"""Upgrade a legacy module dict to canonical snapshot shape.
|
|
|
|
Idempotent: returns ``raw`` unchanged if it already has both
|
|
``library_id`` and ``instance`` keys (the legacy "frozen" shape).
|
|
|
|
SPA-picked modules don't carry ``library_id`` (the picked uuid IS
|
|
the library id, stored as ``id``), but they may carry an
|
|
``instance`` dict with user overrides (option enable/disable,
|
|
weight overrides). Merge those onto the fresh-instance defaults
|
|
instead of stomping them with ``_fresh_instance()`` below.
|
|
"""
|
|
if "instance" in raw and "library_id" in raw:
|
|
return raw
|
|
|
|
# Capture caller-supplied instance early so we can merge it with
|
|
# defaults once the rest of the snapshot is built. Missing → empty.
|
|
raw_instance = raw.get("instance")
|
|
user_instance = raw_instance if isinstance(raw_instance, dict) else {}
|
|
|
|
payload: dict[str, Any]
|
|
if raw.get("type") == "fixed_values":
|
|
# Two shapes can land here:
|
|
#
|
|
# 1. Inline-created (legacy): `entries: [{variable_name, value}]`
|
|
# with no `payload.values`. Translate entries → values.
|
|
# 2. Library-picked (post-5.5.4): `payload: {values: [...]}` with
|
|
# an empty `entries: []` (the field is always present so the
|
|
# ContextWidget conflict-scanner doesn't have to null-check).
|
|
# Use `payload` directly — translating its empty entries
|
|
# would zero out a perfectly good values list.
|
|
raw_payload = raw.get("payload") or {}
|
|
legacy_entries = raw.get("entries") or []
|
|
if legacy_entries and not raw_payload.get("values"):
|
|
payload = {
|
|
"values": [
|
|
{
|
|
"id": f"val_{i:04x}",
|
|
"name": e.get("variable_name", ""),
|
|
"value": e.get("value", ""),
|
|
}
|
|
for i, e in enumerate(legacy_entries)
|
|
],
|
|
}
|
|
else:
|
|
payload = raw_payload
|
|
else:
|
|
payload = raw.get("payload", {})
|
|
|
|
meta = raw.get("meta") or {}
|
|
instance = _fresh_instance()
|
|
instance.update(user_instance)
|
|
return {
|
|
"library_id": raw.get("library_id"),
|
|
"library_snapshot_at": raw.get("library_snapshot_at"),
|
|
"library_version_at_snapshot": raw.get("library_version_at_snapshot"),
|
|
"type": raw.get("type", "fixed_values"),
|
|
"name": raw.get("name") or meta.get("name", ""),
|
|
"category_id": raw.get("category_id"),
|
|
"payload": payload,
|
|
"instance": instance,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Transitive-ref walker (spec §2.7, §2.8, §2.10, §6)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class WalkOverflow(TypedDict):
|
|
"""Walker-recorded anomaly. Returned alongside the partial snapshot
|
|
bundle; never aborts the walk. Spec §2.8."""
|
|
uuid: str
|
|
reason: Literal["max_depth", "cycle_detected", "missing_target"]
|
|
|
|
|
|
@dataclass
|
|
class WalkResult:
|
|
"""Return value of `walk_transitive_refs`. Two fields, two concerns:
|
|
`snapshots` is the materialised catalog dict; `walk_overflow` lists
|
|
every uuid the walker could not include (cycle, depth, miss)."""
|
|
snapshots: dict[str, SnapshotEntry] = field(default_factory=dict)
|
|
walk_overflow: list[WalkOverflow] = field(default_factory=list)
|
|
|
|
|
|
def extract_at_uuid_refs(payload: dict[str, Any]) -> set[str]:
|
|
"""Collect all ``@{8hex}`` refs referenced inside a payload's
|
|
string-typed leaves. Uses ``tokenize_text`` exclusively — only
|
|
canonical 8-hex-char refs (spec §2.4) are followed.
|
|
|
|
Surface-agnostic: collects from any string in the dict/list tree;
|
|
the caller decides which module types have their payloads walked
|
|
(spec §2.7)."""
|
|
refs: set[str] = set()
|
|
|
|
def _scan(value: Any) -> None:
|
|
if isinstance(value, str):
|
|
for tok in tokenize_text(value):
|
|
if tok.kind == TokenKind.REF:
|
|
uid = (tok.meta or {}).get("uuid")
|
|
if isinstance(uid, str):
|
|
refs.add(uid)
|
|
elif isinstance(value, dict):
|
|
for v in value.values():
|
|
_scan(v)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
_scan(item)
|
|
|
|
_scan(payload)
|
|
return refs
|
|
|
|
|
|
def _row_to_snapshot_entry(
|
|
row: dict[str, Any], source: SnapshotSource,
|
|
) -> SnapshotEntry:
|
|
"""Convert a repository row dict to a canonical SnapshotEntry.
|
|
|
|
Post migration 004 the row's `id` IS the canonical 8-hex uuid;
|
|
there is no separate `uuid` field. The snapshot schema still
|
|
names the field `uuid` because that's the published wire format
|
|
embedded in workflow JSON — we just source it from `row["id"]`.
|
|
"""
|
|
return {
|
|
"snapshot_version": 1,
|
|
"uuid": row["id"],
|
|
"type": row["type"],
|
|
"name": row["name"],
|
|
"payload": row["payload"],
|
|
"payload_hash": row.get("payload_hash") or payload_hash(row["payload"]),
|
|
"source": source,
|
|
}
|
|
|
|
|
|
def ref_seed_uuids(modules: Iterable[Any] | None) -> set[str]:
|
|
"""Every ``@{uuid}`` target referenced by ANY module in a node's list.
|
|
|
|
A catalog is synthesised from wildcard modules only, and
|
|
``walk_transitive_refs`` follows refs found in wildcard payloads only. That
|
|
covers a ref authored inside a wildcard option, but ``@{uuid}`` is legal on
|
|
other surfaces too — most notably a derivation rule's action value, and the
|
|
per-instance ``action_value_overrides`` / ``condition_value_overrides`` the
|
|
edit modal writes when a user retargets a branch on the canvas.
|
|
|
|
Refs on those surfaces were never discovered, so their target reached the
|
|
catalog only by coincidence — when the user happened to ALSO have that
|
|
wildcard picked in the same node. Removing the wildcard from the node (a
|
|
reasonable thing to do once a derivation references it by uuid) dropped it
|
|
with nothing to put it back, and every such ref resolved to ``unknown_ref``
|
|
→ empty string no matter what the live library held.
|
|
|
|
Seeds therefore come from each module's ``payload`` AND its ``instance``:
|
|
the instance is where a canvas-side retarget lives, and it is not part of
|
|
the library payload the walker would otherwise see.
|
|
|
|
Tolerant of malformed rows — a corrupt widget value must never break a
|
|
graph run.
|
|
"""
|
|
seeds: set[str] = set()
|
|
for module in modules or []:
|
|
if not isinstance(module, dict):
|
|
continue
|
|
for key in ("payload", "instance"):
|
|
value = module.get(key)
|
|
if isinstance(value, dict):
|
|
seeds |= extract_at_uuid_refs(value)
|
|
return seeds
|
|
|
|
|
|
def walk_transitive_refs(
|
|
root_uuids: Iterable[str],
|
|
*,
|
|
fetch_module: Callable[[str], dict[str, Any] | None],
|
|
max_depth: int = 8,
|
|
) -> WalkResult:
|
|
"""Lazy transitive walker. Spec §2.10.
|
|
|
|
Starts from ``root_uuids``, follows ``@{8hex}`` refs in wildcard payloads
|
|
only (spec §2.7), and returns a ``WalkResult`` containing every reachable
|
|
SnapshotEntry plus a list of overflow records (cycle, depth, missing).
|
|
|
|
Cycle detection uses a per-path ancestor frozenset carried with each BFS
|
|
queue entry. When we pop ``(uuid, depth, parent, ancestors)`` and find
|
|
``uuid`` already in ``ancestors``, we record ``cycle_detected`` and skip
|
|
without emitting the node. This means the node that *caused* the cycle
|
|
reference is already snapshotted (it was emitted when we first reached it
|
|
on a shorter path), so both sides of a two-node cycle appear in snapshots.
|
|
|
|
Engine isolation invariant: this function does NOT import from ``engine.db``
|
|
or ``wp_api``; the caller injects a ``fetch_module(uuid) -> row | None``
|
|
callable. WP_Context graph nodes never call this — they receive a
|
|
pre-walked dict embedded in workflow JSON.
|
|
"""
|
|
result = WalkResult()
|
|
# parent_map: uuid -> list of direct parent uuids (accumulated across BFS)
|
|
parent_map: dict[str, list[str]] = {}
|
|
seen_overflow: set[tuple[str, str]] = set()
|
|
|
|
def _record_overflow(uuid: str, reason: str) -> None:
|
|
key = (uuid, reason)
|
|
if key in seen_overflow:
|
|
return
|
|
seen_overflow.add(key)
|
|
result.walk_overflow.append(
|
|
{"uuid": uuid, "reason": reason}, # type: ignore[arg-type]
|
|
)
|
|
|
|
# Queue items: (uuid, depth, parent_uuid_or_None, ancestor_frozenset)
|
|
# ancestor_frozenset contains all uuids on the path from a root to *before*
|
|
# this node — used to detect back-edges (cycles).
|
|
queue: list[tuple[str, int, str | None, frozenset[str]]] = []
|
|
|
|
for u in root_uuids:
|
|
queue.append((u, 0, None, frozenset()))
|
|
|
|
while queue:
|
|
uuid, depth, parent, ancestors = queue.pop(0)
|
|
|
|
# Track parentage before any short-circuit so multi-parent deps
|
|
# accumulate even if we have already snapshotted the child.
|
|
if parent is not None:
|
|
parent_map.setdefault(uuid, [])
|
|
if parent not in parent_map[uuid]:
|
|
parent_map[uuid].append(parent)
|
|
|
|
# Cycle: this uuid appears on the path from a root to here.
|
|
if uuid in ancestors:
|
|
_record_overflow(uuid, "cycle_detected")
|
|
continue
|
|
|
|
# Already snapshotted on a shorter BFS path — skip emission but
|
|
# parent tracking above already ran, so multi-parent deps accumulate.
|
|
if uuid in result.snapshots:
|
|
continue
|
|
|
|
if depth > max_depth:
|
|
_record_overflow(uuid, "max_depth")
|
|
continue
|
|
|
|
row = fetch_module(uuid)
|
|
if row is None:
|
|
_record_overflow(uuid, "missing_target")
|
|
continue
|
|
|
|
# Determine source based on whether this uuid has a parent.
|
|
if parent is None:
|
|
source: SnapshotSource = {"kind": "user"}
|
|
else:
|
|
source = {"kind": "dep", "parent_uuids": list(parent_map[uuid])}
|
|
result.snapshots[uuid] = _row_to_snapshot_entry(row, source)
|
|
|
|
# Only walk wildcard payloads — spec §2.7.
|
|
if row.get("type") != "wildcard":
|
|
continue
|
|
|
|
new_ancestors = ancestors | {uuid}
|
|
for child_uuid in extract_at_uuid_refs(row["payload"]):
|
|
queue.append((child_uuid, depth + 1, uuid, new_ancestors))
|
|
|
|
# After walk: re-stamp every dep's parent list (BFS may have accumulated
|
|
# additional parents after first emission).
|
|
for uuid, parents in parent_map.items():
|
|
if uuid not in result.snapshots:
|
|
continue
|
|
existing = result.snapshots[uuid]
|
|
if existing["source"]["kind"] == "dep":
|
|
existing["source"]["parent_uuids"] = list(parents)
|
|
|
|
return result
|