feat: measure what the shader does to H3's audio
drive.py saves H3's decoded audio as FLAC, takes --prompt to pick material with a percussive, tonal or broadband sound, and --shade-non-spatial to paint the audio stream too. analyze_audio.py reads each run against its own prompt and seed's strength-0 control and separates what painting the audio latent did from what reached the sound through the picture alone. Prompt and audio painting now reach run names, so a rerun under either does not skip a run it already holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Æmotion Studio
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
18a909b88b
commit
dd8b599fd2
@@ -0,0 +1,196 @@
|
||||
"""
|
||||
Measure what shader noise did to H3's audio stream, from runs recorded by drive.py.
|
||||
|
||||
python verification/blend/analyze_audio.py MANIFEST [MANIFEST ...] [--json FILE]
|
||||
|
||||
Every run is read against the strength-0 control for its own prompt and seed, because
|
||||
level and brightness vary far more between prompts than between strengths.
|
||||
|
||||
H3 denoises both streams in one packed sequence, so a shader painted only on the
|
||||
picture reaches the sound through joint attention anyway. Runs of both arms -- the
|
||||
same strengths with and without shade_non_spatial -- therefore belong in one manifest,
|
||||
and the summary keeps them apart: only the gap between the two arms is what painting
|
||||
the audio latent itself did.
|
||||
|
||||
rms dB loudness, over both channels, so it does not move with stereo width
|
||||
centroid spectral centre of mass in Hz: how bright the sound is
|
||||
flatness geometric over arithmetic mean of the power spectrum, per frame. A pure
|
||||
tone reads 0; white noise reads about 0.56, not 1, because a single frame's
|
||||
periodogram is exponentially distributed. A fall means the model was pushed
|
||||
toward tonal content and away from broadband texture
|
||||
L/R corr correlation between the two channels. The shader reads the audio latent's
|
||||
stereo pair as the height of a 2 x T grid, and create_coordinate_grid
|
||||
samples height 2 at linspace(0, 1, 2) -- the two opposite edges of the
|
||||
field -- so the two channels are drawn as far apart as the pattern allows.
|
||||
If that matters, this falls as strength rises.
|
||||
env Hz the dominant rate of the amplitude envelope, with the peak's height over
|
||||
the band median in brackets. Unmodulated noise stays under about 5 at these
|
||||
clip lengths, while a shallow 30% modulation already reads near 50 -- so
|
||||
under 5 the rate is whatever noise won and means nothing. This is the
|
||||
measure that separates the shader imposing structure over time from it
|
||||
only recolouring timbre.
|
||||
|
||||
Spectral measures are taken per channel and averaged, so stereo decorrelation cannot
|
||||
leak into them.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import soundfile
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import common # noqa: E402
|
||||
|
||||
FRAME, HOP = 2048, 512
|
||||
# The envelope is read in windows far longer than the hop, so the carrier's own ripple
|
||||
# is smoothed away instead of aliasing down into the band. ENV_WINDOW is 32 ms and the
|
||||
# hop leaves 125 Hz of envelope, well above twice the top of the band. The band itself
|
||||
# is the range a listener hears as pulse or swell rather than as pitch or as one fade.
|
||||
ENV_WINDOW, ENV_HOP = 1024, 256
|
||||
ENV_BAND = (0.5, 20.0)
|
||||
|
||||
|
||||
def framed(x, size, hop):
|
||||
count = 1 + max(0, (len(x) - size) // hop)
|
||||
index = np.arange(size)[None, :] + hop * np.arange(count)[:, None]
|
||||
return x[index] * np.hanning(size)
|
||||
|
||||
|
||||
def spectral(x, rate):
|
||||
"""Centroid in Hz and spectral flatness, averaged over frames."""
|
||||
power = np.abs(np.fft.rfft(framed(x, FRAME, HOP), axis=-1)) ** 2 + 1e-12
|
||||
freqs = np.fft.rfftfreq(FRAME, 1.0 / rate)
|
||||
centroid = (power * freqs).sum(axis=-1) / power.sum(axis=-1)
|
||||
flatness = np.exp(np.log(power).mean(axis=-1)) / power.mean(axis=-1)
|
||||
return float(centroid.mean()), float(flatness.mean())
|
||||
|
||||
|
||||
def envelope_rate(x, rate):
|
||||
"""Dominant modulation rate in Hz, and how far its peak stands above the band."""
|
||||
env = np.sqrt((framed(x, ENV_WINDOW, ENV_HOP) ** 2).mean(axis=-1))
|
||||
if len(env) < 8:
|
||||
return 0.0, 0.0
|
||||
level = env.mean()
|
||||
if level <= 0:
|
||||
return 0.0, 0.0
|
||||
# Against its own level, so the rate does not depend on how loud the run came out.
|
||||
env = env / level - 1.0
|
||||
if env.std() < 1e-6: # a steady envelope has no rate, and its peak would be noise
|
||||
return 0.0, 0.0
|
||||
magnitude = np.abs(np.fft.rfft(env * np.hanning(len(env))))
|
||||
freqs = np.fft.rfftfreq(len(env), ENV_HOP / rate)
|
||||
band = (freqs >= ENV_BAND[0]) & (freqs <= ENV_BAND[1])
|
||||
if not band.any():
|
||||
return 0.0, 0.0
|
||||
magnitude, freqs = magnitude[band], freqs[band]
|
||||
peak = int(magnitude.argmax())
|
||||
return float(freqs[peak]), float(magnitude[peak] / (np.median(magnitude) + 1e-12))
|
||||
|
||||
|
||||
def measure(path):
|
||||
audio, rate = soundfile.read(path, dtype="float32", always_2d=True)
|
||||
rms = float(np.sqrt((audio ** 2).mean()))
|
||||
per_channel = [spectral(audio[:, c], rate) for c in range(audio.shape[1])]
|
||||
rates = [envelope_rate(audio[:, c], rate) for c in range(audio.shape[1])]
|
||||
if audio.shape[1] == 2:
|
||||
left, right = audio[:, 0] - audio[:, 0].mean(), audio[:, 1] - audio[:, 1].mean()
|
||||
lr = float((left * right).sum() / (np.linalg.norm(left) * np.linalg.norm(right) + 1e-12))
|
||||
else:
|
||||
lr = 1.0
|
||||
return {
|
||||
"rms_db": 20 * np.log10(rms + 1e-12),
|
||||
"centroid": float(np.mean([c for c, _ in per_channel])),
|
||||
"flatness": float(np.mean([f for _, f in per_channel])),
|
||||
"lr_corr": lr,
|
||||
"env_hz": float(np.mean([h for h, _ in rates])),
|
||||
"env_peak": float(np.mean([p for _, p in rates])),
|
||||
}
|
||||
|
||||
|
||||
# A control's env Hz is whatever noise won, so a change in it means nothing; the rate
|
||||
# is only worth reading on its own, next to the peak that says whether it is real.
|
||||
DELTAS = ("rms_db", "centroid", "flatness", "lr_corr")
|
||||
ABSOLUTE = ("env_hz", "env_peak")
|
||||
|
||||
|
||||
def line(label, m):
|
||||
return (f" {label:22s} {m['rms_db']:7.1f} {m['centroid']:10.0f} {m['flatness']:10.4f} "
|
||||
f"{m['lr_corr']:10.2f} {m['env_hz']:8.1f} ({m['env_peak']:.1f})")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("manifests", nargs="+")
|
||||
ap.add_argument("--json", default=None, help="write every measurement here as well")
|
||||
args = ap.parse_args()
|
||||
|
||||
rows = {}
|
||||
for path in args.manifests:
|
||||
for row in common.read_manifest(path):
|
||||
rows.setdefault(row["name"], row)
|
||||
runs = [r for r in rows.values() if r.get("audio")]
|
||||
if not runs:
|
||||
raise SystemExit("no run in these manifests saved audio; only H3 does, and only since "
|
||||
"drive.py started writing FLAC")
|
||||
|
||||
measured = {}
|
||||
for row in runs:
|
||||
if not Path(row["audio"]).exists():
|
||||
raise SystemExit(f"{row['name']}: {row['audio']} is gone")
|
||||
measured[row["name"]] = measure(row["audio"])
|
||||
|
||||
groups = defaultdict(list)
|
||||
for row in runs:
|
||||
groups[(row.get("prompt_kind", "forge"), row["seed"])].append(row)
|
||||
|
||||
deltas = defaultdict(list)
|
||||
for (prompt, seed), members in sorted(groups.items()):
|
||||
members.sort(key=lambda r: (r["strength"], r["name"]))
|
||||
controls = [r for r in members if r["strength"] == 0]
|
||||
if not controls:
|
||||
raise SystemExit(f"{prompt}, seed {seed}: no strength-0 control to read against")
|
||||
control = measured[controls[0]["name"]]
|
||||
print(f"\n {prompt}, seed {seed}" + " " * 9 + "rms dB centroid flatness L/R corr env Hz (peak)")
|
||||
for row in members:
|
||||
m = measured[row["name"]]
|
||||
painted = "audio" if row.get("shade_non_spatial") else "picture only"
|
||||
print(line(f"{row['strength']:<5g} {painted}", m))
|
||||
if row["strength"] != 0:
|
||||
summary = {k: m[k] - control[k] for k in DELTAS}
|
||||
summary.update({k: m[k] for k in ABSOLUTE})
|
||||
deltas[(prompt, row["strength"], bool(row.get("shade_non_spatial")))].append(summary)
|
||||
|
||||
if deltas:
|
||||
print("\n averaged over seeds: change from each one's own control, and the rate as it stands")
|
||||
print(" prompt strength painted n d rms dB d centroid d flatness d L/R corr env Hz (peak)")
|
||||
means = {}
|
||||
for key, items in sorted(deltas.items()):
|
||||
prompt, strength, painted = key
|
||||
means[key] = {k: float(np.mean([d[k] for d in items])) for k in DELTAS + ABSOLUTE}
|
||||
m = means[key]
|
||||
print(f" {prompt:10s} {strength:<9g} {'audio' if painted else 'picture':9s} {len(items):<4d} "
|
||||
f"{m['rms_db']:+8.1f} {m['centroid']:+11.0f} {m['flatness']:+11.4f} "
|
||||
f"{m['lr_corr']:+11.2f} {m['env_hz']:8.1f} ({m['env_peak']:.1f})")
|
||||
|
||||
paired = [k for k in means if k[2] and (k[0], k[1], False) in means]
|
||||
if paired:
|
||||
print("\n what painting the audio latent did, over painting the picture alone")
|
||||
print(" prompt strength d rms dB d centroid d flatness d L/R corr d env Hz")
|
||||
for key in sorted(paired):
|
||||
audio, picture = means[key], means[(key[0], key[1], False)]
|
||||
gap = {k: audio[k] - picture[k] for k in DELTAS + ABSOLUTE}
|
||||
print(f" {key[0]:10s} {key[1]:<14g} {gap['rms_db']:+8.1f} {gap['centroid']:+11.0f} "
|
||||
f"{gap['flatness']:+11.4f} {gap['lr_corr']:+11.2f} {gap['env_hz']:+11.1f}")
|
||||
|
||||
if args.json:
|
||||
Path(args.json).write_text(json.dumps(measured, indent=1) + "\n")
|
||||
print("\nwrote", args.json)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,6 +5,10 @@ drift apart.
|
||||
Every run holds the shader to one configuration and varies six things: the seed,
|
||||
the strength, the travel mode, phase_shift and noise_scale for the "streets", and
|
||||
the shader type. A strength-0 run has no shader, so it is shared by every type.
|
||||
|
||||
The prompt and whether the audio stream is painted are fixed for a whole matrix
|
||||
rather than varied within one, but they still change the output, so run_name carries
|
||||
them too.
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
@@ -34,7 +38,8 @@ def load_pack():
|
||||
import helpers # noqa: F401
|
||||
|
||||
|
||||
def shader_inputs(seed, strength, travel, phase, scale, shader=DEFAULT_SHADER):
|
||||
def shader_inputs(seed, strength, travel, phase, scale, shader=DEFAULT_SHADER,
|
||||
shade_non_spatial=False):
|
||||
"""Direct node inputs shared by every run. Sampler settings are per model."""
|
||||
return {
|
||||
"seed": seed, "denoise": 1.0, "sequential_stages": 1, "injection_stages": 0,
|
||||
@@ -43,7 +48,8 @@ def shader_inputs(seed, strength, travel, phase, scale, shader=DEFAULT_SHADER):
|
||||
"color_scheme": "none", "noise_scale": scale, "octaves": 2.0, "warp_strength": 0.7,
|
||||
"shape_mask_strength": 1.0, "phase_shift": phase, "color_intensity": 0.8,
|
||||
"sampling_mode": "standard", "preset": "custom", "travel_mode": travel,
|
||||
"normalize_strength": True, "stage_progression": "uniform", "shade_non_spatial": False,
|
||||
"normalize_strength": True, "stage_progression": "uniform",
|
||||
"shade_non_spatial": shade_non_spatial,
|
||||
}
|
||||
|
||||
|
||||
@@ -52,18 +58,36 @@ def shader_of(strength, shader):
|
||||
return "none" if strength == 0 else shader
|
||||
|
||||
|
||||
def audio_of(strength, shade_non_spatial):
|
||||
"""Same reason: with no shader to paint, the control is shared with the picture runs."""
|
||||
return bool(shade_non_spatial) and strength != 0
|
||||
|
||||
|
||||
def run_key(row):
|
||||
return (row["seed"], row["strength"], row["travel"], row["phase"], row["scale"],
|
||||
shader_of(row["strength"], row.get("shader", DEFAULT_SHADER)))
|
||||
|
||||
|
||||
def run_name(seed, strength, travel, phase, scale, shader=DEFAULT_SHADER):
|
||||
def run_name(seed, strength, travel, phase, scale, shader=DEFAULT_SHADER,
|
||||
shade_non_spatial=False, prompt=None):
|
||||
"""
|
||||
What a run is filed under. Anything that changes the output has to reach this
|
||||
name, or a rerun skips the run it already holds and one setting stands in for
|
||||
another. A setting every earlier run shared is left out, so those runs keep the
|
||||
names they were recorded under: the default shader type, the model's own prompt,
|
||||
and painting the picture alone.
|
||||
"""
|
||||
# Two decimals unless that would merge distinct strengths, so 0.001 is not named 0.00.
|
||||
shown = f"{strength:.2f}" if round(strength, 2) == strength else f"{strength:g}"
|
||||
name = f"s{seed}_{travel}_{shown}_p{phase:.1f}_n{scale:.1f}"
|
||||
# The default type keeps the names its runs were recorded under.
|
||||
shader = shader_of(strength, shader)
|
||||
return name if shader in ("none", DEFAULT_SHADER) else f"{name}_{shader}"
|
||||
if shader not in ("none", DEFAULT_SHADER):
|
||||
name = f"{name}_{shader}"
|
||||
if prompt:
|
||||
name = f"{name}_{prompt}"
|
||||
if audio_of(strength, shade_non_spatial):
|
||||
name = f"{name}_audio"
|
||||
return name
|
||||
|
||||
|
||||
def read_manifest(path):
|
||||
|
||||
+74
-23
@@ -2,12 +2,17 @@
|
||||
Run a blend-measurement matrix through a running ComfyUI server.
|
||||
|
||||
python verification/blend/drive.py {sd15,h3,krea2} --label NAME [--strengths 0.25,0.5]
|
||||
[--street-strengths 0.5] [--jump 0.5] [--seeds 8888,1234] [--shader-types a,b] [--dry-run]
|
||||
[--street-strengths 0.5] [--jump 0.5] [--seeds 8888,1234] [--shader-types a,b]
|
||||
[--prompt forge] [--shade-non-spatial] [--length 124] [--dry-run]
|
||||
|
||||
Strength 0 is always included for every seed: it is what every distance is read
|
||||
against. Each run appends a line to <out>/<model>_<label>.jsonl with where its latent
|
||||
and image or video landed, and a rerun skips what the manifest already holds.
|
||||
|
||||
H3 runs also save the decoded audio as FLAC, for analyze_audio.py and for listening.
|
||||
`--shade-non-spatial` paints the audio stream as well as the picture, and `--prompt`
|
||||
picks material with a different audio character to hear it against.
|
||||
|
||||
The server has to have the pack loaded, SD 1.5, MiniMax H3 or Krea 2 installed
|
||||
under the file names below, and nothing else queued: interleaving models makes it
|
||||
swap them for every prompt.
|
||||
@@ -28,15 +33,31 @@ FORGE = ("Close-up of a blacksmith hammering a glowing orange horseshoe on an an
|
||||
"Loud rhythmic metallic clangs of hammer on steel, crackling fire, faint bellows hiss.")
|
||||
RIDER = ("A dramatic digital painting of a Persian woman in her 40s, with warm olive-brown skin, "
|
||||
"dark hair, and an amused smirking expression, seated atop a big red elephant.")
|
||||
# H3 generates sound with the picture, so its prompts come in three audio characters:
|
||||
# percussive, tonal and broadband. Each still has to give a scene worth judging.
|
||||
CELLO = ("A cellist alone in a candlelit stone chapel, bow drawn slowly across the strings, "
|
||||
"dust drifting through a shaft of light from a high window. "
|
||||
"One sustained cello note, low and resonant, swelling and fading in the hall.")
|
||||
RAIN = ("Heavy rain hammering a corrugated tin roof over a deserted night market, water "
|
||||
"sheeting off the edges into puddles under a single bare bulb. "
|
||||
"A dense unbroken roar of rainfall on metal, no music and no voices.")
|
||||
|
||||
PROMPTS = {"sd15": {"fisherman": PORTRAIT},
|
||||
"h3": {"forge": FORGE, "cello": CELLO, "rain": RAIN},
|
||||
"krea2": {"rider": RIDER}}
|
||||
# The prompt every run of a model used before there was a choice, left out of run names.
|
||||
DEFAULT_PROMPT = {"sd15": "fisherman", "h3": "forge", "krea2": "rider"}
|
||||
# er_sde/beta57 for the shader work; euler/normal is what SD 1.5 was measured with.
|
||||
SAMPLER = {"sd15": ("euler", "normal"), "h3": ("er_sde", "beta57"), "krea2": ("er_sde", "beta57")}
|
||||
|
||||
|
||||
def sd15_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
s = dict(common.shader_inputs(seed, strength, travel, phase, scale, shader), steps=20, cfg=7.0,
|
||||
sampler_name="euler", scheduler="normal",
|
||||
def sd15_graph(prefix, seed, strength, travel, phase, scale, shader, size, opts):
|
||||
s = dict(common.shader_inputs(seed, strength, travel, phase, scale, shader, opts["shade_non_spatial"]),
|
||||
steps=20, cfg=7.0, sampler_name=opts["sampler"], scheduler=opts["scheduler"],
|
||||
model=["1", 0], positive=["2", 0], negative=["3", 0], latent_image=["4", 0])
|
||||
return {
|
||||
"1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "v1-5-pruned-emaonly-fp16.safetensors"}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["1", 1], "text": PORTRAIT}},
|
||||
"2": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["1", 1], "text": opts["prompt"]}},
|
||||
"3": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["1", 1], "text": "blurry, low quality, watermark"}},
|
||||
"4": {"class_type": "EmptyLatentImage", "inputs": {"width": size["width"], "height": size["height"], "batch_size": 1}},
|
||||
"5": {"class_type": "ShaderNoiseKSamplerDirect", "inputs": s},
|
||||
@@ -46,9 +67,9 @@ def sd15_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
}
|
||||
|
||||
|
||||
def h3_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
s = dict(common.shader_inputs(seed, strength, travel, phase, scale, shader), steps=8, cfg=1.0,
|
||||
sampler_name="res_multistep", scheduler="simple",
|
||||
def h3_graph(prefix, seed, strength, travel, phase, scale, shader, size, opts):
|
||||
s = dict(common.shader_inputs(seed, strength, travel, phase, scale, shader, opts["shade_non_spatial"]),
|
||||
steps=8, cfg=1.0, sampler_name=opts["sampler"], scheduler=opts["scheduler"],
|
||||
model=["2", 0], positive=["6", 0], negative=["7", 0], latent_image=["6", 1])
|
||||
return {
|
||||
"1": {"class_type": "UNETLoader", "inputs": {"unet_name": "10Eros_Max_h3_TURBO-hybrid_beta5_int8.safetensors", "weight_dtype": "default"}},
|
||||
@@ -56,7 +77,7 @@ def h3_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
"3": {"class_type": "CLIPLoader", "inputs": {"clip_name": "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", "type": "minimax", "device": "default"}},
|
||||
"4": {"class_type": "VAELoader", "inputs": {"vae_name": "minimax_h3_video_vae_int8_convrot.safetensors"}},
|
||||
"5": {"class_type": "VAELoader", "inputs": {"vae_name": "minimax_h3_audio_vae_fp32.safetensors"}},
|
||||
"6": {"class_type": "MiniMaxH3ImageToVideo", "inputs": {"clip": ["3", 0], "vae": ["4", 0], "prompt": FORGE,
|
||||
"6": {"class_type": "MiniMaxH3ImageToVideo", "inputs": {"clip": ["3", 0], "vae": ["4", 0], "prompt": opts["prompt"],
|
||||
"width": size["width"], "height": size["height"], "length": size["length"]}},
|
||||
"7": {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": ["6", 0]}},
|
||||
"8": {"class_type": "ShaderNoiseKSamplerDirect", "inputs": s},
|
||||
@@ -67,19 +88,21 @@ def h3_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
"12": {"class_type": "VAEDecodeAudio", "inputs": {"samples": ["8", 0], "vae": ["5", 0]}},
|
||||
"13": {"class_type": "CreateVideo", "inputs": {"images": ["11", 0], "fps": 24.0, "audio": ["12", 0]}},
|
||||
"14": {"class_type": "SaveVideo", "inputs": {"video": ["13", 0], "filename_prefix": prefix, "format": "auto", "format.codec": "auto"}},
|
||||
# The muxed mp4 is for watching; the FLAC is what analyze_audio.py measures.
|
||||
"15": {"class_type": "SaveAudioAdvanced", "inputs": {"audio": ["12", 0], "filename_prefix": prefix, "format": "flac"}},
|
||||
}
|
||||
|
||||
|
||||
def krea2_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
def krea2_graph(prefix, seed, strength, travel, phase, scale, shader, size, opts):
|
||||
# The user's own Krea 2 workflow, with the Direct node in the KSampler's place.
|
||||
s = dict(common.shader_inputs(seed, strength, travel, phase, scale, shader), steps=8, cfg=1.0,
|
||||
sampler_name="er_sde", scheduler="beta57",
|
||||
s = dict(common.shader_inputs(seed, strength, travel, phase, scale, shader, opts["shade_non_spatial"]),
|
||||
steps=8, cfg=1.0, sampler_name=opts["sampler"], scheduler=opts["scheduler"],
|
||||
model=["1", 0], positive=["5", 0], negative=["6", 0], latent_image=["7", 0])
|
||||
return {
|
||||
"1": {"class_type": "UNETLoader", "inputs": {"unet_name": "lustifyNSFWCheckpoint_v10Krea2_2996235.safetensors", "weight_dtype": "default"}},
|
||||
"2": {"class_type": "CLIPLoader", "inputs": {"clip_name": "krea2UncensoredLLMCLIP_v10_int8.safetensors", "type": "krea2", "device": "default"}},
|
||||
"3": {"class_type": "VAELoader", "inputs": {"vae_name": "qwen_image_vae.safetensors"}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["2", 0], "text": RIDER}},
|
||||
"4": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["2", 0], "text": opts["prompt"]}},
|
||||
"5": {"class_type": "ConditioningKrea2Rebalance", "inputs": {"conditioning": ["4", 0], "multiplier": 3.0,
|
||||
"per_layer_weights": "1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.5,5.0,1.1,4.0,1.0"}},
|
||||
"6": {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": ["4", 0]}},
|
||||
@@ -91,7 +114,7 @@ def krea2_graph(prefix, seed, strength, travel, phase, scale, shader, size):
|
||||
}
|
||||
|
||||
|
||||
GRAPHS = {"sd15": (sd15_graph, PORTRAIT), "h3": (h3_graph, FORGE), "krea2": (krea2_graph, RIDER)}
|
||||
GRAPHS = {"sd15": sd15_graph, "h3": h3_graph, "krea2": krea2_graph}
|
||||
|
||||
|
||||
def floats(text):
|
||||
@@ -124,7 +147,7 @@ def submit(server, graph):
|
||||
|
||||
def wait(server, prompt_id, output_dir):
|
||||
"""Block until the prompt finishes; return its saved files by kind."""
|
||||
kinds = {".latent": "latent", ".png": "image", ".mp4": "video"}
|
||||
kinds = {".latent": "latent", ".png": "image", ".mp4": "video", ".flac": "audio"}
|
||||
while True:
|
||||
time.sleep(1.5)
|
||||
history = json.load(urllib.request.urlopen(f"{server}/history/{prompt_id}", timeout=60))
|
||||
@@ -154,38 +177,66 @@ def main():
|
||||
ap.add_argument("--jump", default="", help="strengths to also run with travel_mode jump")
|
||||
ap.add_argument("--seeds", default="", help="default: " + ", ".join(f"{m} {common.SEEDS[m]}" for m in common.SEEDS))
|
||||
ap.add_argument("--shader-types", default=common.DEFAULT_SHADER, help="comma-separated shader types to run")
|
||||
ap.add_argument("--prompt", default=None, help="prompt key; default: "
|
||||
+ ", ".join(f"{m} {DEFAULT_PROMPT[m]} of {sorted(PROMPTS[m])}" for m in sorted(PROMPTS)))
|
||||
ap.add_argument("--shade-non-spatial", action="store_true",
|
||||
help="paint the streams with no picture in them too -- on H3 that is the audio")
|
||||
ap.add_argument("--length", type=int, default=None, help="frame count, overriding the model's default")
|
||||
ap.add_argument("--sampler", default=None, help="default: " + ", ".join(f"{m} {SAMPLER[m][0]}" for m in sorted(SAMPLER)))
|
||||
ap.add_argument("--scheduler", default=None, help="default: " + ", ".join(f"{m} {SAMPLER[m][1]}" for m in sorted(SAMPLER)))
|
||||
ap.add_argument("--server", default="http://127.0.0.1:8188")
|
||||
ap.add_argument("--comfy-output", default=os.path.expanduser("~/ComfyUI/output"))
|
||||
ap.add_argument("--out", default=None, help="manifest directory (default: <comfy-output>/snk_measure)")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
prompt_kind = args.prompt or DEFAULT_PROMPT[args.model]
|
||||
if prompt_kind not in PROMPTS[args.model]:
|
||||
raise SystemExit(f"{args.model} has no prompt {prompt_kind!r}; "
|
||||
f"try {', '.join(sorted(PROMPTS[args.model]))}")
|
||||
sampler, scheduler = SAMPLER[args.model]
|
||||
opts = {"prompt": PROMPTS[args.model][prompt_kind], "shade_non_spatial": args.shade_non_spatial,
|
||||
"sampler": args.sampler or sampler, "scheduler": args.scheduler or scheduler}
|
||||
seeds = [int(s) for s in args.seeds.split(",") if s.strip()] or list(common.SEEDS[args.model])
|
||||
shaders = [s.strip() for s in args.shader_types.split(",") if s.strip()]
|
||||
runs = plan(args.model, floats(args.strengths), floats(args.street_strengths), floats(args.jump), seeds, shaders)
|
||||
|
||||
# Fixed for the whole matrix, but still part of what a run is: a rerun under a
|
||||
# different prompt or with the audio painted must not skip the picture run it
|
||||
# already holds. The model's own prompt and picture-only runs keep their old names.
|
||||
def named(run):
|
||||
return common.run_name(*run, shade_non_spatial=args.shade_non_spatial,
|
||||
prompt=None if prompt_kind == DEFAULT_PROMPT[args.model] else prompt_kind)
|
||||
|
||||
out = Path(args.out or os.path.join(args.comfy_output, "snk_measure"))
|
||||
manifest = out / f"{args.model}_{args.label}.jsonl"
|
||||
done = {r["name"] for r in common.read_manifest(manifest)} if manifest.exists() else set()
|
||||
todo = [r for r in runs if common.run_name(*r) not in done]
|
||||
todo = [r for r in runs if named(r) not in done]
|
||||
print(f"{args.model}: {len(runs)} runs planned, {len(todo)} to do -> {manifest}")
|
||||
print(f" {prompt_kind}, {opts['sampler']}/{opts['scheduler']}, "
|
||||
f"audio {'painted' if args.shade_non_spatial else 'left alone'}")
|
||||
if args.dry_run:
|
||||
for r in todo:
|
||||
print(" ", common.run_name(*r))
|
||||
print(" ", named(r))
|
||||
return
|
||||
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
graph, prompt = GRAPHS[args.model]
|
||||
size = common.SIZE[args.model]
|
||||
graph = GRAPHS[args.model]
|
||||
size = dict(common.SIZE[args.model])
|
||||
if args.length:
|
||||
size["length"] = args.length
|
||||
for run in todo:
|
||||
name = common.run_name(*run)
|
||||
name = named(run)
|
||||
started = time.time()
|
||||
prefix = f"snk_measure/{args.model}/{args.label}/{name}"
|
||||
files = wait(args.server, submit(args.server, graph(prefix, *run, size)), args.comfy_output)
|
||||
files = wait(args.server, submit(args.server, graph(prefix, *run, size, opts)), args.comfy_output)
|
||||
seed, strength, travel, phase, scale, shader = run
|
||||
row = dict(name=name, model=args.model, seed=seed, strength=strength, travel=travel, phase=phase,
|
||||
scale=scale, shader=common.shader_of(strength, shader), prompt=prompt, **size,
|
||||
scale=scale, shader=common.shader_of(strength, shader), prompt=opts["prompt"],
|
||||
prompt_kind=prompt_kind, shade_non_spatial=common.audio_of(strength, args.shade_non_spatial),
|
||||
sampler=opts["sampler"], scheduler=opts["scheduler"], **size,
|
||||
latent=files.get("latent"), image=files.get("image"), video=files.get("video"),
|
||||
seconds=round(time.time() - started, 1))
|
||||
audio=files.get("audio"), seconds=round(time.time() - started, 1))
|
||||
with open(manifest, "a") as fh:
|
||||
fh.write(json.dumps(row) + "\n")
|
||||
print(name, row["seconds"], flush=True)
|
||||
|
||||
Reference in New Issue
Block a user