ComfyUI-Replicate-Select v0.1.0 — 14 kuratierte Modelle + ReplicateCustom
Bild (Seedream 4.5, Nano Banana Pro), Video (Kling v3, Veo 3.1, Seedance 2.0 Fast), Upscale (SUPIR, Real-ESRGAN), TTS (MiniMax 2.8 HD, Inworld TTS 2, ElevenLabs v3), Lipsync (VEED Fabric, Pruna Avatar, Sync 2 Pro, HeyGen Precision). Architektur nach Runware-Vorbild: nodes.json + Generator + Escape-Hatch. Katalog-Refresh ohne Token via scripts/build_catalog.py (public Schema-Seiten). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K33rz4XBNkq2gKoQov5D37
This commit is contained in:
Pharma-Lobby
@@ -0,0 +1,43 @@
|
||||
# ComfyUI-Replicate-Select
|
||||
|
||||
Kuratierte Replicate-Modelle als ComfyUI-Nodes — schlank statt Katalog-Flut.
|
||||
Architektur nach dem Vorbild von [ComfyUI-Runware](https://github.com/Runware/ComfyUI-Runware):
|
||||
`nodes.json` (Katalog) + Generator + `ReplicateCustom` als Escape-Hatch.
|
||||
|
||||
## Modelle (14)
|
||||
|
||||
| Kategorie | Nodes |
|
||||
|---|---|
|
||||
| image | Seedream 4.5 · Nano Banana Pro |
|
||||
| video | Kling v3 Video · Veo 3.1 · Seedance 2.0 Fast |
|
||||
| upscale | SUPIR · Real-ESRGAN |
|
||||
| tts | MiniMax Speech 2.8 HD · Inworld TTS 2 · ElevenLabs v3 |
|
||||
| lipsync | VEED Fabric 1.0 · Pruna Video Avatar · Sync Lipsync 2 Pro · HeyGen Lipsync Precision |
|
||||
|
||||
Dazu: **Replicate Custom** — beliebige Modell-ID + JSON-Input.
|
||||
|
||||
## Auth
|
||||
|
||||
`REPLICATE_API_TOKEN` als Umgebungsvariable (auf Modal: Secret `mcp-keys`).
|
||||
Kein Key im Node, keine Key-Dateien.
|
||||
|
||||
## Outputs
|
||||
|
||||
- **image** → IMAGE-Tensor (direkt weiterverdrahtbar)
|
||||
- **audio** → AUDIO (ComfyUI-nativ, z.B. → SaveAudio oder → Lipsync-Nodes via Upload)
|
||||
- **video** → MP4 im Output-Ordner, Node gibt den Pfad als STRING zurück
|
||||
|
||||
## Katalog aktualisieren / Modelle hinzufügen
|
||||
|
||||
```bash
|
||||
# Modell in scripts/build_catalog.py → META eintragen, dann:
|
||||
python scripts/build_catalog.py
|
||||
```
|
||||
|
||||
Zieht die Input-Schemas von den öffentlichen Replicate-Schema-Seiten — kein Token nötig.
|
||||
|
||||
## Hinweise
|
||||
|
||||
- Bild-Inputs werden als PNG hochgeladen (klein → data-URI, groß → Replicate Files-API)
|
||||
- Audio-Inputs (Lipsync) akzeptieren ComfyUI-AUDIO und werden als WAV hochgeladen
|
||||
- Video-Inputs erwarten eine URL (STRING) — Replicate braucht abrufbare Dateien
|
||||
@@ -0,0 +1,3 @@
|
||||
from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
|
||||
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
|
||||
Binary file not shown.
+1486
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,262 @@
|
||||
"""ComfyUI-Replicate-Select — kuratierte Replicate-Modelle als ComfyUI-Nodes.
|
||||
|
||||
Architektur nach Runware-Vorbild: nodes.json (Katalog) + Generator + Custom-Node.
|
||||
Auth: REPLICATE_API_TOKEN aus der Umgebung (Modal-Secret mcp-keys).
|
||||
Katalog aktualisieren: python scripts/build_catalog.py
|
||||
"""
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
CATALOG_PATH = Path(__file__).parent / "nodes.json"
|
||||
CATEGORY_ROOT = "Replicate Select"
|
||||
DATA_URI_LIMIT = 200 * 1024 # größere Dateien gehen über die Files-API
|
||||
|
||||
|
||||
# ---------- Replicate-Client ----------
|
||||
|
||||
def _client():
|
||||
import replicate
|
||||
if not os.environ.get("REPLICATE_API_TOKEN"):
|
||||
raise RuntimeError(
|
||||
"REPLICATE_API_TOKEN fehlt. In Modal: Secret 'mcp-keys' prüfen; "
|
||||
"lokal: export REPLICATE_API_TOKEN=r8_..."
|
||||
)
|
||||
return replicate
|
||||
|
||||
|
||||
def _upload_bytes(data: bytes, filename: str, content_type: str) -> str:
|
||||
"""Datei zu Replicate hochladen (Files-API) oder als data-URI inline."""
|
||||
if len(data) <= DATA_URI_LIMIT:
|
||||
b64 = base64.b64encode(data).decode()
|
||||
return f"data:{content_type};base64,{b64}"
|
||||
import replicate
|
||||
f = replicate.files.create(io.BytesIO(data), {"filename": filename, "content_type": content_type})
|
||||
return f.urls["get"]
|
||||
|
||||
|
||||
# ---------- Medien-Konvertierung ----------
|
||||
|
||||
def _image_to_uri(image_tensor) -> str:
|
||||
"""ComfyUI IMAGE (B,H,W,C float 0-1) → PNG-URI (erstes Bild im Batch)."""
|
||||
from PIL import Image
|
||||
arr = (image_tensor[0].cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
|
||||
img = Image.fromarray(arr)
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
return _upload_bytes(buf.getvalue(), "input.png", "image/png")
|
||||
|
||||
|
||||
def _audio_to_uri(audio: dict) -> str:
|
||||
"""ComfyUI AUDIO ({'waveform': (B,C,T), 'sample_rate': int}) → WAV-URI."""
|
||||
import torchaudio
|
||||
buf = io.BytesIO()
|
||||
waveform = audio["waveform"][0] # (C, T)
|
||||
torchaudio.save(buf, waveform.cpu(), audio["sample_rate"], format="wav")
|
||||
return _upload_bytes(buf.getvalue(), "input.wav", "audio/wav")
|
||||
|
||||
|
||||
def _download(url_or_file) -> bytes:
|
||||
"""Replicate-Output (FileOutput oder URL-String) → Bytes."""
|
||||
if hasattr(url_or_file, "read"):
|
||||
return url_or_file.read()
|
||||
import urllib.request
|
||||
with urllib.request.urlopen(str(url_or_file), timeout=600) as resp:
|
||||
return resp.read()
|
||||
|
||||
|
||||
def _to_image_tensor(data: bytes):
|
||||
from PIL import Image
|
||||
img = Image.open(io.BytesIO(data)).convert("RGB")
|
||||
arr = np.asarray(img).astype(np.float32) / 255.0
|
||||
return torch.from_numpy(arr)[None,]
|
||||
|
||||
|
||||
def _to_audio(data: bytes) -> dict:
|
||||
import torchaudio
|
||||
waveform, sr = torchaudio.load(io.BytesIO(data))
|
||||
return {"waveform": waveform[None,], "sample_rate": sr}
|
||||
|
||||
|
||||
def _save_video(data: bytes, model_id: str) -> str:
|
||||
"""Video in den ComfyUI-Output-Ordner schreiben, Pfad zurückgeben."""
|
||||
try:
|
||||
import folder_paths
|
||||
out_dir = Path(folder_paths.get_output_directory())
|
||||
except Exception:
|
||||
out_dir = Path("output")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
slug = model_id.replace("/", "_")
|
||||
path = out_dir / f"replicate_{slug}_{int(time.time())}.mp4"
|
||||
path.write_bytes(data)
|
||||
return str(path)
|
||||
|
||||
|
||||
def _first_output(result):
|
||||
"""Replicate gibt je nach Modell FileOutput, Liste oder dict zurück."""
|
||||
if isinstance(result, (list, tuple)):
|
||||
return result[0] if result else None
|
||||
if isinstance(result, dict):
|
||||
for key in ("video", "audio", "image", "output", "url"):
|
||||
if key in result:
|
||||
return result[key]
|
||||
return next(iter(result.values()), None)
|
||||
return result
|
||||
|
||||
|
||||
# ---------- Node-Generierung ----------
|
||||
|
||||
def _comfy_input(entry: dict):
|
||||
kind = entry["kind"]
|
||||
tooltip = {"tooltip": entry["description"]} if entry.get("description") else {}
|
||||
if kind == "combo":
|
||||
return ([str(o) for o in entry["options"]], {"default": str(entry.get("default", "")), **tooltip})
|
||||
if kind == "bool":
|
||||
return ("BOOLEAN", {"default": entry.get("default", False), **tooltip})
|
||||
if kind == "int":
|
||||
return ("INT", {"default": int(entry.get("default") or 0),
|
||||
"min": int(max(entry.get("min", -2**31), -2**31)),
|
||||
"max": int(min(entry.get("max", 2**31 - 1), 2**31 - 1)), **tooltip})
|
||||
if kind == "float":
|
||||
return ("FLOAT", {"default": float(entry.get("default") or 0.0),
|
||||
"min": float(entry.get("min", -1e9)), "max": float(entry.get("max", 1e9)),
|
||||
"step": 0.01, **tooltip})
|
||||
if kind == "image":
|
||||
return ("IMAGE", tooltip)
|
||||
if kind == "audio":
|
||||
return ("AUDIO", tooltip)
|
||||
# string / uri_string
|
||||
return ("STRING", {"default": str(entry.get("default") or ""),
|
||||
"multiline": entry.get("multiline", False), **tooltip})
|
||||
|
||||
|
||||
def _build_arguments(model_def: dict, kwargs: dict) -> dict:
|
||||
args = {}
|
||||
for entry in model_def["inputs"]:
|
||||
name = entry["name"]
|
||||
value = kwargs.get(name)
|
||||
if value is None:
|
||||
continue
|
||||
kind = entry["kind"]
|
||||
if kind == "image":
|
||||
uri = _image_to_uri(value)
|
||||
args[name] = [uri] if entry.get("is_list") else uri
|
||||
elif kind == "audio":
|
||||
args[name] = _audio_to_uri(value)
|
||||
elif kind == "combo":
|
||||
args[name] = value
|
||||
elif kind in ("string", "uri_string"):
|
||||
if str(value).strip():
|
||||
args[name] = value
|
||||
else:
|
||||
args[name] = value
|
||||
return args
|
||||
|
||||
|
||||
_RETURN_BY_OUTPUT = {
|
||||
"image": (("IMAGE",), ("image",)),
|
||||
"audio": (("AUDIO",), ("audio",)),
|
||||
"video": (("STRING",), ("video_path",)),
|
||||
}
|
||||
|
||||
|
||||
def _make_node(model_def: dict):
|
||||
output_kind = model_def["output"]
|
||||
return_types, return_names = _RETURN_BY_OUTPUT[output_kind]
|
||||
|
||||
required, optional = {}, {}
|
||||
for entry in model_def["inputs"]:
|
||||
target = required if entry["required"] else optional
|
||||
target[entry["name"]] = _comfy_input(entry)
|
||||
|
||||
class ReplicateModelNode:
|
||||
CATEGORY = f"{CATEGORY_ROOT}/{model_def['category']}"
|
||||
FUNCTION = "run"
|
||||
RETURN_TYPES = return_types
|
||||
RETURN_NAMES = return_names
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
spec = {"required": dict(required)}
|
||||
if optional:
|
||||
spec["optional"] = dict(optional)
|
||||
return spec
|
||||
|
||||
def run(self, **kwargs):
|
||||
replicate = _client()
|
||||
args = _build_arguments(model_def, kwargs)
|
||||
print(f"[Replicate Select] {model_def['id']} ← {list(args.keys())}")
|
||||
result = replicate.run(model_def["id"], input=args)
|
||||
output = _first_output(result)
|
||||
if output is None:
|
||||
raise RuntimeError(f"{model_def['id']}: leere Antwort")
|
||||
data = _download(output)
|
||||
if output_kind == "image":
|
||||
return (_to_image_tensor(data),)
|
||||
if output_kind == "audio":
|
||||
return (_to_audio(data),)
|
||||
return (_save_video(data, model_def["id"]),)
|
||||
|
||||
ReplicateModelNode.__name__ = "Replicate_" + model_def["id"].replace("/", "_").replace("-", "_").replace(".", "_")
|
||||
return ReplicateModelNode
|
||||
|
||||
|
||||
class ReplicateCustom:
|
||||
"""Beliebiges Replicate-Modell per ID aufrufen — der Escape-Hatch."""
|
||||
|
||||
CATEGORY = CATEGORY_ROOT
|
||||
FUNCTION = "run"
|
||||
RETURN_TYPES = ("STRING", "IMAGE")
|
||||
RETURN_NAMES = ("raw_output", "image")
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model_id": ("STRING", {"default": "owner/model", "tooltip": "z.B. bytedance/seedream-4.5"}),
|
||||
"input_json": ("STRING", {"multiline": True, "default": '{\n "prompt": ""\n}',
|
||||
"tooltip": "Input-Parameter als JSON"}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"tooltip": "Wird als 'image'-Parameter angehängt, falls gesetzt"}),
|
||||
},
|
||||
}
|
||||
|
||||
def run(self, model_id, input_json, image=None):
|
||||
replicate = _client()
|
||||
args = json.loads(input_json)
|
||||
if image is not None:
|
||||
args["image"] = _image_to_uri(image)
|
||||
result = replicate.run(model_id.strip(), input=args)
|
||||
output = _first_output(result)
|
||||
raw = str(output)
|
||||
image_out = torch.zeros(1, 1, 1, 3)
|
||||
try:
|
||||
data = _download(output)
|
||||
if data[:8].startswith((b"\x89PNG", b"\xff\xd8")):
|
||||
image_out = _to_image_tensor(data)
|
||||
else:
|
||||
raw = f"{raw}\n(bytes: {len(data)})"
|
||||
except Exception:
|
||||
pass
|
||||
return (raw, image_out)
|
||||
|
||||
|
||||
def _load_catalog() -> dict:
|
||||
with open(CATALOG_PATH, encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"ReplicateCustom": ReplicateCustom}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"ReplicateCustom": "Replicate Custom (beliebige Modell-ID)"}
|
||||
|
||||
for _model in _load_catalog()["models"]:
|
||||
_cls = _make_node(_model)
|
||||
NODE_CLASS_MAPPINGS[_cls.__name__] = _cls
|
||||
NODE_DISPLAY_NAME_MAPPINGS[_cls.__name__] = f"{_model['name']} (Replicate)"
|
||||
@@ -0,0 +1,9 @@
|
||||
[project]
|
||||
name = "comfyui-replicate-select"
|
||||
description = "Kuratierte Replicate-Modelle als ComfyUI-Nodes (Bild, Video, Upscale, TTS, Lipsync) + Custom-Node für beliebige Modell-IDs."
|
||||
version = "0.1.0"
|
||||
dependencies = ["replicate>=1.0.7"]
|
||||
|
||||
[tool.comfy]
|
||||
PublisherId = "pharma-lobby"
|
||||
DisplayName = "Replicate Select"
|
||||
@@ -0,0 +1 @@
|
||||
replicate>=1.0.7
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Baut nodes.json aus den öffentlichen Replicate-Schema-Seiten (kein API-Token nötig).
|
||||
|
||||
Aufruf: python scripts/build_catalog.py
|
||||
Schreibt: nodes.json im Repo-Root.
|
||||
|
||||
Neue Modelle: unten in META eintragen, Script laufen lassen, fertig.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
# Modell-ID → (Anzeigename, Kategorie, Output-Art: image|video|audio)
|
||||
META = {
|
||||
"bytedance/seedream-4.5": ("Seedream 4.5", "image", "image"),
|
||||
"google/nano-banana-pro": ("Nano Banana Pro", "image", "image"),
|
||||
"kwaivgi/kling-v3-video": ("Kling v3 Video", "video", "video"),
|
||||
"google/veo-3.1": ("Veo 3.1", "video", "video"),
|
||||
"bytedance/seedance-2.0-fast": ("Seedance 2.0 Fast", "video", "video"),
|
||||
"cjwbw/supir": ("SUPIR Upscale", "upscale", "image"),
|
||||
"cjwbw/real-esrgan": ("Real-ESRGAN Upscale", "upscale", "image"),
|
||||
"minimax/speech-2.8-hd": ("MiniMax Speech 2.8 HD","tts", "audio"),
|
||||
"inworld/realtime-tts-2": ("Inworld TTS 2", "tts", "audio"),
|
||||
"elevenlabs/v3": ("ElevenLabs v3", "tts", "audio"),
|
||||
"veed/fabric-1.0": ("VEED Fabric 1.0", "lipsync", "video"),
|
||||
"prunaai/p-video-avatar": ("Pruna Video Avatar", "lipsync", "video"),
|
||||
"sync/lipsync-2-pro": ("Sync Lipsync 2 Pro", "lipsync", "video"),
|
||||
"heygen/lipsync-precision": ("HeyGen Lipsync Precision", "lipsync", "video"),
|
||||
}
|
||||
|
||||
|
||||
def fetch_schema(model_id: str) -> dict | None:
|
||||
"""Schema aus dem eingebetteten JSON der öffentlichen Schema-Seite ziehen."""
|
||||
req = urllib.request.Request(
|
||||
f"https://replicate.com/{model_id}/api/schema",
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
)
|
||||
html = urllib.request.urlopen(req, timeout=30).read().decode()
|
||||
for m in re.finditer(r'<script[^>]*type="application/json"[^>]*>(.*?)</script>', html, re.DOTALL):
|
||||
try:
|
||||
data = json.loads(m.group(1))
|
||||
except Exception:
|
||||
continue
|
||||
version = data.get("version") or data
|
||||
schema = version.get("_extras", {}).get("dereferenced_openapi_schema") or {}
|
||||
schemas = schema.get("components", {}).get("schemas", {})
|
||||
if schemas.get("Input"):
|
||||
return {"input": schemas["Input"], "version_id": version.get("id")}
|
||||
return None
|
||||
|
||||
|
||||
def classify_uri(name: str, is_list: bool) -> dict:
|
||||
"""URI-Inputs nach Medientyp einsortieren."""
|
||||
lname = name.lower()
|
||||
if any(k in lname for k in ("audio", "voice_sample", "speech")):
|
||||
return {"kind": "audio"}
|
||||
if any(k in lname for k in ("video", "motion")):
|
||||
return {"kind": "uri_string"} # Video-Inputs: URL als String (zu groß für Upload-Tricks)
|
||||
if any(k in lname for k in ("image", "img", "frame", "mask", "reference")):
|
||||
return {"kind": "image", "is_list": is_list}
|
||||
return {"kind": "uri_string"}
|
||||
|
||||
|
||||
def distill(model_id: str, input_schema: dict) -> dict:
|
||||
props = input_schema.get("properties", {})
|
||||
required = set(input_schema.get("required", []))
|
||||
inputs = []
|
||||
for name, p in props.items():
|
||||
if "allOf" in p and p["allOf"]:
|
||||
merged = dict(p["allOf"][0])
|
||||
merged.update({k: v for k, v in p.items() if k != "allOf"})
|
||||
p = merged
|
||||
entry = {
|
||||
"name": name,
|
||||
"order": p.get("x-order", 999),
|
||||
"required": name in required,
|
||||
"description": (p.get("description") or "")[:200],
|
||||
}
|
||||
t = p.get("type")
|
||||
fmt = p.get("format", "")
|
||||
items = p.get("items", {}) if isinstance(p.get("items"), dict) else {}
|
||||
is_uri = fmt == "uri" or items.get("format") == "uri"
|
||||
is_list = t == "array"
|
||||
lname = name.lower()
|
||||
|
||||
if p.get("enum"):
|
||||
entry["kind"] = "combo"
|
||||
entry["options"] = p["enum"]
|
||||
entry["default"] = p.get("default", p["enum"][0])
|
||||
elif t == "boolean":
|
||||
entry["kind"] = "bool"
|
||||
entry["default"] = bool(p.get("default", False))
|
||||
elif t == "integer":
|
||||
entry["kind"] = "int"
|
||||
entry["default"] = p.get("default", 0)
|
||||
entry["min"] = p.get("minimum", -2**31)
|
||||
entry["max"] = p.get("maximum", 2**31 - 1)
|
||||
elif t == "number":
|
||||
entry["kind"] = "float"
|
||||
entry["default"] = p.get("default", 0.0)
|
||||
entry["min"] = p.get("minimum", -1e9)
|
||||
entry["max"] = p.get("maximum", 1e9)
|
||||
elif is_uri:
|
||||
entry.update(classify_uri(name, is_list))
|
||||
else:
|
||||
entry["kind"] = "string"
|
||||
entry["default"] = p.get("default") or ""
|
||||
if any(k in lname for k in ("prompt", "text", "instruction", "script")):
|
||||
entry["multiline"] = True
|
||||
inputs.append(entry)
|
||||
inputs.sort(key=lambda e: (not e["required"], e["order"]))
|
||||
name, cat, out = META[model_id]
|
||||
return {"id": model_id, "name": name, "category": cat, "output": out, "inputs": inputs}
|
||||
|
||||
|
||||
def main():
|
||||
models = []
|
||||
for model_id in META:
|
||||
try:
|
||||
fetched = fetch_schema(model_id)
|
||||
except Exception as exc:
|
||||
print(f"✗ {model_id}: {exc}")
|
||||
continue
|
||||
if not fetched:
|
||||
print(f"✗ {model_id}: kein Schema gefunden")
|
||||
continue
|
||||
entry = distill(model_id, fetched["input"])
|
||||
models.append(entry)
|
||||
print(f"✓ {model_id}: {len(entry['inputs'])} inputs")
|
||||
time.sleep(1)
|
||||
|
||||
out_path = Path(__file__).resolve().parent.parent / "nodes.json"
|
||||
out_path.write_text(json.dumps({"models": models}, indent=1, ensure_ascii=False))
|
||||
print(f"→ {out_path} ({len(models)}/{len(META)} Modelle)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user