v1.3.0: sidecar metadata + thumbnail + orphan migration
HashVaultSave now writes {hash_key}.json (label, timestamps, payload summary) and {hash_key}.thumb.png (256px preview) alongside the .pt, inside the existing FileLock. New optional label input on Save node, does not factor into the hash. DeterministicHashVault bumps last_accessed_at on cache hit (silent no-op for pre-v1.3.0 entries). folder_paths import is tolerant of being outside ComfyUI so the sidecar helpers are reusable from tooling.
tools/migrate_hash_vault.py backfills sidecars for pre-v1.3.0 entries, using file mtime as created_at and "(legacy)" as label.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jeremie louvaert
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
eda9826431
commit
84e074766d
@@ -32,6 +32,26 @@ ComfyUI's native caching often breaks with external API nodes (dynamic timestamp
|
||||
- **Device-Portable Caching:** All tensors are saved to CPU and loaded with `map_location="cpu"`, so cache files work regardless of GPU configuration.
|
||||
- **Atomic Writes:** Cache files are written to a temp file first, then atomically replaced — preventing corruption from interrupted writes.
|
||||
- **Concurrent-Safe:** File locking on every cache read/write operation.
|
||||
- **Sidecar Metadata (v1.3.0):** Every saved entry gets a `{hash_key}.json` sidecar with human-readable `label`, `created_at`, `last_accessed_at`, and payload summary. Image outputs also get a `{hash_key}.thumb.png` 256px preview. Sidecar data is decoupled from the hash, so editing a `label` never invalidates cache. This is what the upcoming Hash Vault Browser indexes.
|
||||
|
||||
### Sidecar Label Input
|
||||
|
||||
The Save Result node has an optional `label` input (v1.3.0+). Wire a human-readable string like `"Alec Soth / Songbook / full"` and the Hash Vault Browser will surface it when you need to find a specific past run. The label is stored in the sidecar JSON only, so changing it never breaks existing cache hits.
|
||||
|
||||
### Migrating pre-v1.3.0 entries
|
||||
|
||||
Existing `.pt` files have no sidecar. To backfill:
|
||||
|
||||
```bash
|
||||
# From the pack root, using the ComfyUI embedded Python:
|
||||
python tools/migrate_hash_vault.py --dry-run # preview
|
||||
python tools/migrate_hash_vault.py # execute
|
||||
|
||||
# Override the vault location:
|
||||
python tools/migrate_hash_vault.py --vault-dir "F:/path/to/output/hash_vault"
|
||||
```
|
||||
|
||||
Orphan entries are labelled `(legacy)` by default and assigned `created_at` from file mtime.
|
||||
|
||||
## How to Use the Hash Vault
|
||||
|
||||
@@ -91,6 +111,8 @@ All data is stored under your ComfyUI output directory:
|
||||
| `output/api_metrics/api_transactions.jsonl` | Append-only audit log with timestamps |
|
||||
| `output/api_metrics/api_costs_archive_*.json` | Archived ledgers from budget resets |
|
||||
| `output/hash_vault/*.pt` | Cached API outputs (PyTorch format) |
|
||||
| `output/hash_vault/*.json` | Sidecar metadata: label, timestamps, payload summary (v1.3.0+) |
|
||||
| `output/hash_vault/*.thumb.png` | 256px preview for image outputs (v1.3.0+) |
|
||||
|
||||
## Installation
|
||||
|
||||
|
||||
+172
-3
@@ -3,10 +3,16 @@ import json
|
||||
import hashlib
|
||||
import time
|
||||
import torch
|
||||
import folder_paths
|
||||
from decimal import Decimal
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
import folder_paths
|
||||
except ImportError:
|
||||
# Allow this module to be imported outside ComfyUI (e.g. by tools/migrate_hash_vault.py)
|
||||
# so the sidecar helpers remain reusable. Node methods still require ComfyUI to run.
|
||||
folder_paths = None
|
||||
|
||||
try:
|
||||
from filelock import FileLock
|
||||
except ImportError:
|
||||
@@ -20,6 +26,12 @@ except ImportError:
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
_PIL_AVAILABLE = True
|
||||
except ImportError:
|
||||
_PIL_AVAILABLE = False
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
# UTILITIES
|
||||
# ------------------------------------------------------------------------
|
||||
@@ -30,6 +42,150 @@ class AnyType(str):
|
||||
|
||||
any_type = AnyType("*")
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
# SIDECAR METADATA HELPERS (v1.3.0)
|
||||
# Every vault entry has a .json sidecar + optional .thumb.png alongside the .pt.
|
||||
# This is what the Hash Vault Browser reads; .pt itself is opaque and expensive to load.
|
||||
# ------------------------------------------------------------------------
|
||||
|
||||
THUMBNAIL_MAX_SIZE = 256
|
||||
SIDECAR_SUFFIX = ".json"
|
||||
THUMBNAIL_SUFFIX = ".thumb.png"
|
||||
|
||||
|
||||
def _find_image_tensor(data):
|
||||
"""Walk an arbitrary api_output and return the first tensor that looks like a ComfyUI image.
|
||||
|
||||
ComfyUI image convention: float tensor of shape [B, H, W, C] with C in {1, 3, 4}, values in [0, 1].
|
||||
Returns None if nothing qualifies.
|
||||
"""
|
||||
if isinstance(data, torch.Tensor):
|
||||
t = data
|
||||
if t.ndim == 4 and t.shape[-1] in (1, 3, 4):
|
||||
return t
|
||||
if t.ndim == 3 and t.shape[-1] in (1, 3, 4):
|
||||
return t
|
||||
return None
|
||||
if isinstance(data, dict):
|
||||
# Skip "samples" (latent) — not a visible image
|
||||
for key, value in data.items():
|
||||
if key == "samples":
|
||||
continue
|
||||
found = _find_image_tensor(value)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
if isinstance(data, (list, tuple)):
|
||||
for item in data:
|
||||
found = _find_image_tensor(item)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _generate_thumbnail(tensor, output_path):
|
||||
"""Write a thumbnail PNG for an image tensor. Returns True on success, False otherwise."""
|
||||
if not _PIL_AVAILABLE:
|
||||
return False
|
||||
try:
|
||||
t = tensor
|
||||
if t.ndim == 4:
|
||||
t = t[0] # first batch item
|
||||
if t.ndim != 3 or t.shape[-1] not in (1, 3, 4):
|
||||
return False
|
||||
|
||||
arr = t.detach().cpu().clamp(0, 1).float().numpy()
|
||||
arr = (arr * 255.0).round().astype("uint8")
|
||||
|
||||
if arr.shape[-1] == 1:
|
||||
arr = arr.squeeze(-1)
|
||||
mode = "L"
|
||||
elif arr.shape[-1] == 3:
|
||||
mode = "RGB"
|
||||
else:
|
||||
mode = "RGBA"
|
||||
|
||||
img = Image.fromarray(arr, mode=mode)
|
||||
img.thumbnail((THUMBNAIL_MAX_SIZE, THUMBNAIL_MAX_SIZE), Image.LANCZOS)
|
||||
|
||||
tmp_path = output_path + ".tmp"
|
||||
img.save(tmp_path, "PNG", optimize=True)
|
||||
os.replace(tmp_path, output_path)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"⚠️ [API Vault] Thumbnail generation failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _summarize_payload(data):
|
||||
"""Produce a lightweight JSON-safe description of api_output for the browser."""
|
||||
if isinstance(data, torch.Tensor):
|
||||
return {
|
||||
"kind": "tensor",
|
||||
"shape": list(data.shape),
|
||||
"dtype": str(data.dtype).replace("torch.", ""),
|
||||
}
|
||||
if isinstance(data, dict):
|
||||
return {
|
||||
"kind": "dict",
|
||||
"keys": sorted([str(k) for k in data.keys()])[:16],
|
||||
"size": len(data),
|
||||
}
|
||||
if isinstance(data, (list, tuple)):
|
||||
return {
|
||||
"kind": type(data).__name__,
|
||||
"length": len(data),
|
||||
"first_item": _summarize_payload(data[0]) if len(data) > 0 else None,
|
||||
}
|
||||
return {"kind": type(data).__name__}
|
||||
|
||||
|
||||
def _write_sidecar(vault_dir, hash_key, cpu_data, label):
|
||||
"""Write {hash_key}.json (+ optional {hash_key}.thumb.png). Caller must already hold the vault lock."""
|
||||
sidecar_path = os.path.join(vault_dir, f"{hash_key}{SIDECAR_SUFFIX}")
|
||||
thumbnail_path = os.path.join(vault_dir, f"{hash_key}{THUMBNAIL_SUFFIX}")
|
||||
now_iso = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
image_tensor = _find_image_tensor(cpu_data)
|
||||
has_thumbnail = False
|
||||
if image_tensor is not None:
|
||||
has_thumbnail = _generate_thumbnail(image_tensor, thumbnail_path)
|
||||
|
||||
sidecar = {
|
||||
"hash_key": hash_key,
|
||||
"label": (label or "").strip(),
|
||||
"created_at": now_iso,
|
||||
"last_accessed_at": now_iso,
|
||||
"payload": _summarize_payload(cpu_data),
|
||||
"thumbnail": f"{hash_key}{THUMBNAIL_SUFFIX}" if has_thumbnail else None,
|
||||
"schema_version": 1,
|
||||
}
|
||||
|
||||
tmp_path = sidecar_path + ".tmp"
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(sidecar, f, indent=2)
|
||||
os.replace(tmp_path, sidecar_path)
|
||||
|
||||
|
||||
def _touch_sidecar(vault_dir, hash_key):
|
||||
"""Bump last_accessed_at on cache hit. Silent no-op if the sidecar is missing (pre-migration entry)."""
|
||||
sidecar_path = os.path.join(vault_dir, f"{hash_key}{SIDECAR_SUFFIX}")
|
||||
if not os.path.exists(sidecar_path):
|
||||
return
|
||||
try:
|
||||
with open(sidecar_path, "r", encoding="utf-8") as f:
|
||||
sidecar = json.load(f)
|
||||
sidecar["last_accessed_at"] = datetime.now(timezone.utc).isoformat()
|
||||
tmp_path = sidecar_path + ".tmp"
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(sidecar, f, indent=2)
|
||||
os.replace(tmp_path, sidecar_path)
|
||||
except Exception as e:
|
||||
# Sidecar update failure must never break a cache hit
|
||||
print(f"⚠️ [API Vault] Failed to update sidecar timestamp for {hash_key[:8]}: {e}")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
# NODE 1: API Cost & Quota Tracker
|
||||
# ------------------------------------------------------------------------
|
||||
@@ -233,6 +389,7 @@ class DeterministicHashVault:
|
||||
# weights_only=False is required because cache may contain dicts/lists
|
||||
# alongside tensors. Safe here — we only load files our own save node wrote.
|
||||
cached_data = torch.load(file_path, map_location="cpu", weights_only=False)
|
||||
_touch_sidecar(vault_dir, hash_key)
|
||||
print(f"🟢 [API Vault] Cache Hit! Hash: {hash_key[:8]}")
|
||||
return (cached_data, 1, hash_key)
|
||||
except Exception as e:
|
||||
@@ -255,6 +412,13 @@ class HashVaultSave:
|
||||
"required": {
|
||||
"hash_key": ("STRING", {"forceInput": True}),
|
||||
"api_output": (any_type,),
|
||||
},
|
||||
"optional": {
|
||||
"label": ("STRING", {
|
||||
"default": "",
|
||||
"multiline": False,
|
||||
"tooltip": "Human-readable label for the Hash Vault Browser (e.g. 'Alec Soth / Songbook / full'). Written to sidecar metadata only — does NOT factor into the hash, so editing this never invalidates cache."
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,7 +441,7 @@ class HashVaultSave:
|
||||
return tuple(self._to_cpu(item) for item in data)
|
||||
return data
|
||||
|
||||
def save_to_vault(self, hash_key, api_output):
|
||||
def save_to_vault(self, hash_key, api_output, label=""):
|
||||
vault_dir = os.path.join(folder_paths.get_output_directory(), "hash_vault")
|
||||
os.makedirs(vault_dir, exist_ok=True)
|
||||
file_path = os.path.join(vault_dir, f"{hash_key}.pt")
|
||||
@@ -292,7 +456,12 @@ class HashVaultSave:
|
||||
torch.save(cpu_data, tmp_path)
|
||||
os.replace(tmp_path, file_path)
|
||||
|
||||
print(f"💾 [API Vault] Saved API output to Vault: {hash_key[:8]}")
|
||||
# Sidecar + thumbnail (v1.3.0) — keep inside the lock so readers
|
||||
# see a consistent (.pt, .json, .thumb.png) triple.
|
||||
_write_sidecar(vault_dir, hash_key, cpu_data, label)
|
||||
|
||||
label_suffix = f" [{label}]" if label else ""
|
||||
print(f"💾 [API Vault] Saved API output to Vault: {hash_key[:8]}{label_suffix}")
|
||||
return (api_output,)
|
||||
|
||||
# ------------------------------------------------------------------------
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "comfyui-api-optimizer"
|
||||
description = "API cost tracking, deterministic hash caching, and lazy execution bypass for cloud API workflows"
|
||||
version = "1.2.2"
|
||||
version = "1.3.0"
|
||||
license = { file = "LICENSE" }
|
||||
|
||||
[project.urls]
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""One-shot backfill for pre-v1.3.0 Hash Vault entries.
|
||||
|
||||
Before v1.3.0, HashVaultSave wrote only `{hash_key}.pt`. v1.3.0 adds a
|
||||
`{hash_key}.json` sidecar (+ optional `{hash_key}.thumb.png`) so the Hash
|
||||
Vault Browser can index entries by label, timestamp, and thumbnail.
|
||||
|
||||
This script scans a vault directory, finds orphan `.pt` files (no matching
|
||||
sidecar), loads each, and writes the missing sidecar + thumbnail using file
|
||||
mtime as `created_at` and `"(legacy)"` as `label`.
|
||||
|
||||
Usage (from the pack root, using the ComfyUI embedded Python):
|
||||
python tools/migrate_hash_vault.py
|
||||
python tools/migrate_hash_vault.py --vault-dir "F:/path/to/output/hash_vault"
|
||||
python tools/migrate_hash_vault.py --dry-run
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
PACK_ROOT = SCRIPT_DIR.parent
|
||||
sys.path.insert(0, str(PACK_ROOT))
|
||||
|
||||
from api_optimizer_nodes import ( # noqa: E402
|
||||
SIDECAR_SUFFIX,
|
||||
THUMBNAIL_SUFFIX,
|
||||
_find_image_tensor,
|
||||
_generate_thumbnail,
|
||||
_summarize_payload,
|
||||
)
|
||||
|
||||
DEFAULT_VAULT_DIR = Path(
|
||||
"F:/ComfyUI_windows_portable_nvidia/ComfyUI_windows_portable/ComfyUI/output/hash_vault"
|
||||
)
|
||||
LEGACY_LABEL = "(legacy)"
|
||||
|
||||
|
||||
def build_sidecar(hash_key: str, cpu_data, label: str, created_iso: str) -> dict:
|
||||
return {
|
||||
"hash_key": hash_key,
|
||||
"label": label,
|
||||
"created_at": created_iso,
|
||||
"last_accessed_at": created_iso,
|
||||
"payload": _summarize_payload(cpu_data),
|
||||
"thumbnail": None, # Filled in after thumbnail write succeeds
|
||||
"schema_version": 1,
|
||||
}
|
||||
|
||||
|
||||
def write_sidecar_atomic(sidecar_path: Path, sidecar: dict) -> None:
|
||||
tmp_path = sidecar_path.with_suffix(sidecar_path.suffix + ".tmp")
|
||||
with open(tmp_path, "w", encoding="utf-8") as f:
|
||||
json.dump(sidecar, f, indent=2)
|
||||
os.replace(tmp_path, sidecar_path)
|
||||
|
||||
|
||||
def migrate_vault(vault_dir: Path, label: str, dry_run: bool) -> dict:
|
||||
if not vault_dir.exists():
|
||||
print(f"[migrate] vault dir does not exist: {vault_dir}")
|
||||
return {"scanned": 0, "skipped": 0, "migrated": 0, "failed": 0}
|
||||
|
||||
stats = {"scanned": 0, "skipped": 0, "migrated": 0, "failed": 0}
|
||||
|
||||
for pt_path in sorted(vault_dir.glob("*.pt")):
|
||||
stats["scanned"] += 1
|
||||
hash_key = pt_path.stem
|
||||
sidecar_path = vault_dir / f"{hash_key}{SIDECAR_SUFFIX}"
|
||||
thumbnail_path = vault_dir / f"{hash_key}{THUMBNAIL_SUFFIX}"
|
||||
|
||||
if sidecar_path.exists():
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
mtime = pt_path.stat().st_mtime
|
||||
created_iso = datetime.fromtimestamp(mtime, tz=timezone.utc).isoformat()
|
||||
|
||||
if dry_run:
|
||||
print(f"[migrate] would backfill {hash_key[:8]} (created {created_iso})")
|
||||
stats["migrated"] += 1
|
||||
continue
|
||||
|
||||
cached_data = torch.load(pt_path, map_location="cpu", weights_only=False)
|
||||
sidecar = build_sidecar(hash_key, cached_data, label, created_iso)
|
||||
|
||||
image_tensor = _find_image_tensor(cached_data)
|
||||
if image_tensor is not None:
|
||||
if _generate_thumbnail(image_tensor, str(thumbnail_path)):
|
||||
sidecar["thumbnail"] = f"{hash_key}{THUMBNAIL_SUFFIX}"
|
||||
|
||||
write_sidecar_atomic(sidecar_path, sidecar)
|
||||
thumb_note = "+thumb" if sidecar["thumbnail"] else "no-thumb"
|
||||
print(f"[migrate] backfilled {hash_key[:8]} ({thumb_note}) created {created_iso}")
|
||||
stats["migrated"] += 1
|
||||
except Exception as e:
|
||||
print(f"[migrate] FAILED {hash_key[:8]}: {e}")
|
||||
stats["failed"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Backfill sidecars for pre-v1.3.0 Hash Vault entries.")
|
||||
parser.add_argument(
|
||||
"--vault-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_VAULT_DIR,
|
||||
help=f"Path to the hash_vault directory (default: {DEFAULT_VAULT_DIR})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--label",
|
||||
type=str,
|
||||
default=LEGACY_LABEL,
|
||||
help=f'Label to assign to orphan entries (default: "{LEGACY_LABEL}")',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="List what would be migrated without writing anything.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"[migrate] vault dir: {args.vault_dir}")
|
||||
print(f"[migrate] label: {args.label!r}")
|
||||
print(f"[migrate] dry run: {args.dry_run}")
|
||||
print("-" * 60)
|
||||
|
||||
stats = migrate_vault(args.vault_dir, args.label, args.dry_run)
|
||||
|
||||
print("-" * 60)
|
||||
print(f"[migrate] scanned: {stats['scanned']}")
|
||||
print(f"[migrate] skipped: {stats['skipped']} (already had sidecar)")
|
||||
print(f"[migrate] migrated: {stats['migrated']}")
|
||||
print(f"[migrate] failed: {stats['failed']}")
|
||||
return 0 if stats["failed"] == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user