Add audio references, Krea controls, and scan effects

This commit is contained in:
Fillip
2026-09-14 09:56:34 -07:00
parent d8edc58da1
commit f949415096
87 changed files with 9868 additions and 227 deletions
+53 -1
View File
@@ -4,6 +4,9 @@ logger = logging.getLogger("fl_fill_nodes")
from . import routes
from .nodes.conditioning.FL_KreaReference import FL_KreaReference, FL_KreaReferenceGuider
from .nodes.ksamplers.FL_KsamplerSEG_Krea import FL_KsamplerSEG_Krea
# AI NODES
from .nodes.ai.FL_Fal_Gemini_ImageEdit import FL_Fal_Gemini_ImageEdit
from .nodes.ai.FL_Fal_GPTImage2_Edit import FL_Fal_GPTImage2_Edit
@@ -43,6 +46,7 @@ from .nodes.api_tools.FL_API_ImageSaver import FL_API_ImageSaver
# AUDIO NODES
from .nodes.audio.FL_Audio_BPM_Analyzer import FL_Audio_BPM_Analyzer
from .nodes.audio.FL_Audio_Beat_Prompt_Schedule import FL_Audio_Beat_Prompt_Schedule
from .nodes.audio.FL_Prompt_Reference_Library import FL_Prompt_Reference_Library
from .nodes.audio.FL_Audio_Prompt_Envelope import FL_Audio_Beat_Prompt_Envelope, FL_Audio_Envelope_Prompt
from .nodes.audio.FL_Audio_Beat_Visualizer import FL_Audio_Beat_Visualizer
from .nodes.audio.FL_Audio_Crop import FL_Audio_Crop
@@ -157,7 +161,7 @@ from .nodes.ksamplers.FL_KsamplerSigma import FL_KsamplerSigma
from .nodes.ksamplers.FL_KsamplerSEG_Regions import FL_KsamplerSEG_Regions
from .nodes.ksamplers.FL_KsamplerSEG_Captioner import FL_KsamplerSEG_Captioner
from .nodes.ksamplers.FL_KsamplerSEG_Encoder import FL_KsamplerSEG_Encoder
from .nodes.ksamplers.FL_KsamplerSEG import FL_KsamplerSEG
from .nodes.ksamplers.FL_KsamplerSEG import FL_KsamplerSEG, FL_KsamplerSEGAdvanced
from .nodes.ksamplers.FL_KsamplerSettings import FL_KsamplerSettings
from .nodes.ksamplers.FL_SamplerStrings import FL_SamplerStrings
from .nodes.ksamplers.FL_SchedulerStrings import FL_SchedulerStrings
@@ -208,6 +212,7 @@ from .nodes.utility.FL_SD_Slices import FL_SDUltimate_Slices
from .nodes.utility.FL_SeparateMasks import FL_SeparateMaskComponents
from .nodes.utility.FL_ShowText import FL_ShowText
from .nodes.utility.FL_StringToLoraName import FL_StringToLoraName
from .nodes.utility.FL_ModelDifferenceLoraSave import FL_ModelDifferenceLoraSave
from .nodes.utility.FL_Switch import FL_Switch
from .nodes.utility.FL_Switch_Big import FL_Switch_Big
from .nodes.utility.FL_SystemCheck import FL_SystemCheck
@@ -220,6 +225,15 @@ from .nodes.vfx.FL_Ascii import FL_Ascii
from .nodes.vfx.FL_DepthBlur import FL_DepthBlur
from .nodes.vfx.FL_Dither import FL_Dither
from .nodes.vfx.FL_Glitch import FL_Glitch
from .nodes.vfx.FL_StreetScan import FL_ScanVideoDetections, FL_StreetScanComposite
from .nodes.vfx.FL_ScanAudioEdit import FL_ScanAudioEdit
from .nodes.vfx.FL_VoxelNormalRelief import FL_VoxelNormalRelief
from .nodes.vfx.FL_LayeredParallax import FL_ParallaxLayer, FL_LayeredParallax, FL_ParallaxDepthSources, FL_ParallaxStackFromBatch
from .nodes.vfx.FL_PosterLayers import FL_PosterLayerPlanner, FL_PosterLayers, FL_PosterLayerAsset, FL_PosterLayerStack
from .nodes.vfx.poster_layer_cache import PosterLayerCache
from comfy_execution.cache_provider import register_cache_provider
from .nodes.vfx.FL_InteractiveScanFX import FL_InteractiveScanFX, FL_ScanAnalysis, FL_ScanVideoSection, FL_ScanVideoShots, FL_ScanAnalysisCollect
from .nodes.vfx.FL_HalfTone import FL_HalftonePattern
from .nodes.vfx.FL_HexagonalPattern import FL_HexagonalPattern
from .nodes.vfx.FL_ImageCollage import FL_ImageCollage
@@ -263,7 +277,12 @@ from .nodes.wip.FL_WanVideoContinuationBlender import FL_WanVideoContinuationBle
from .nodes.wip.FL_ZImageControlNetPatch import FL_ZImageControlNetPatch
from .nodes.wip.FL_LTXVMaskedICLoRAGuide import FL_LTXVMaskedICLoRAGuide
register_cache_provider(PosterLayerCache())
NODE_CLASS_MAPPINGS = {
"FL_KreaReference": FL_KreaReference,
"FL_KreaReferenceGuider": FL_KreaReferenceGuider,
"FL_KsamplerSEG_Krea": FL_KsamplerSEG_Krea,
"FL_SaveWebM": FL_SaveWebM,
"FL_TextOverlayNode": FL_TextOverlayNode,
"FL_ImageBlank": FL_ImageBlank,
@@ -281,6 +300,23 @@ NODE_CLASS_MAPPINGS = {
"FL_ImageAddNoise": FL_ImageAddNoise,
"FL_WordFrequencyGraph": FL_WordFrequencyGraph,
"FL_Glitch": FL_Glitch,
"FL_ScanVideoDetections": FL_ScanVideoDetections,
"FL_StreetScanComposite": FL_StreetScanComposite,
"FL_ScanAudioEdit": FL_ScanAudioEdit,
"FL_VoxelNormalRelief": FL_VoxelNormalRelief,
"FL_ParallaxLayer": FL_ParallaxLayer,
"FL_LayeredParallax": FL_LayeredParallax,
"FL_ParallaxDepthSources": FL_ParallaxDepthSources,
"FL_ParallaxStackFromBatch": FL_ParallaxStackFromBatch,
"FL_PosterLayerPlanner": FL_PosterLayerPlanner,
"FL_PosterLayers": FL_PosterLayers,
"FL_PosterLayerAsset": FL_PosterLayerAsset,
"FL_PosterLayerStack": FL_PosterLayerStack,
"FL_InteractiveScanFX": FL_InteractiveScanFX,
"FL_ScanAnalysis": FL_ScanAnalysis,
"FL_ScanVideoSection": FL_ScanVideoSection,
"FL_ScanVideoShots": FL_ScanVideoShots,
"FL_ScanAnalysisCollect": FL_ScanAnalysisCollect,
"FL_Ripple": FL_Ripple,
"FL_PixelSort": FL_PixelSort,
"FL_HexagonalPattern": FL_HexagonalPattern,
@@ -332,6 +368,7 @@ NODE_CLASS_MAPPINGS = {
"FL_KsamplerSEG_Captioner": FL_KsamplerSEG_Captioner,
"FL_KsamplerSEG_Encoder": FL_KsamplerSEG_Encoder,
"FL_KsamplerSEG": FL_KsamplerSEG,
"FL_KsamplerSEGAdvanced": FL_KsamplerSEGAdvanced,
"FL_FractalKSampler": FL_FractalKSampler,
"FL_UpscaleModel": FL_UpscaleModel,
"FL_SaveCSV": FL_SaveCSV,
@@ -414,6 +451,7 @@ NODE_CLASS_MAPPINGS = {
"FL_GPT_Text": FL_GPT_Text,
"FL_GoogleCloudStorage": FL_GoogleCloudStorage,
"FL_StringToLoraName": FL_StringToLoraName,
"FL_ModelDifferenceLoraSave": FL_ModelDifferenceLoraSave,
"FL_Switch": FL_Switch,
"FL_Switch_Big": FL_Switch_Big,
"FL_PasteByMask": FL_PasteByMask,
@@ -446,6 +484,7 @@ NODE_CLASS_MAPPINGS = {
"FL_SaveRGBAAnimatedWebP": FL_SaveRGBAAnimatedWebP,
"FL_Audio_BPM_Analyzer": FL_Audio_BPM_Analyzer,
"FL_Audio_Beat_Prompt_Schedule": FL_Audio_Beat_Prompt_Schedule,
"FL_Prompt_Reference_Library": FL_Prompt_Reference_Library,
"FL_Audio_Beat_Prompt_Envelope": FL_Audio_Beat_Prompt_Envelope,
"FL_Audio_Envelope_Prompt": FL_Audio_Envelope_Prompt,
"FL_Audio_Beat_Visualizer": FL_Audio_Beat_Visualizer,
@@ -490,6 +529,15 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"FL_ImageAddNoise": "FL Image Add Noise",
"FL_WordFrequencyGraph": "FL Word Frequency Graph",
"FL_Glitch": "FL Glitch",
"FL_ScanVideoDetections": "FL Scan Video Detections",
"FL_StreetScanComposite": "FL Street Scan Composite",
"FL_ScanAudioEdit": "FL Scan Audio Edit",
"FL_VoxelNormalRelief": "FL Voxel Normal Relief",
"FL_ParallaxLayer": "FL Parallax Layer",
"FL_LayeredParallax": "FL Layered Parallax",
"FL_InteractiveScanFX": "FL Interactive Scan FX",
"FL_ScanAnalysis": "FL Scan Analysis",
"FL_ScanVideoSection": "FL Scan Video Section",
"FL_Ripple": "FL Ripple",
"FL_PixelSort": "FL PixelSort",
"FL_HexagonalPattern": "FL Hexagonal Pattern",
@@ -542,6 +590,8 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"FL_KsamplerSEG_Captioner": "FL KSampler SEG Captioner",
"FL_KsamplerSEG_Encoder": "FL KSampler SEG Encoder",
"FL_KsamplerSEG": "FL KSampler SEG",
"FL_KsamplerSEGAdvanced": "FL KSampler SEG Advanced",
"FL_KsamplerSEG_Krea": "FL KSampler SEG Krea",
"FL_FractalKSampler": "FL Fractal KSampler",
"FL_UpscaleModel": "FL Upscale Model",
"FL_SaveCSV": "FL Save CSV",
@@ -624,6 +674,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"FL_GPT_Text": "FL GPT Text",
"FL_GoogleCloudStorage": "FL Google Cloud Storage Uploader",
"FL_StringToLoraName": "FL String To Lora Name",
"FL_ModelDifferenceLoraSave": "FL Model Difference to LoRA",
"FL_Switch": "FL Switch",
"FL_Switch_Big": "FL Switch Big",
"FL_PasteByMask": "FL Paste By Mask",
@@ -658,6 +709,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"FL_SaveRGBAAnimatedWebP": "FL Save RGBA Animated WebP",
"FL_Audio_BPM_Analyzer": "FL Audio BPM Analyzer",
"FL_Audio_Beat_Prompt_Schedule": "FL Audio Beat Prompt Schedule",
"FL_Prompt_Reference_Library": "FL Prompt Reference Library",
"FL_Audio_Beat_Prompt_Envelope": "FL Audio Beat Prompt Envelope",
"FL_Audio_Envelope_Prompt": "FL Audio Envelope Prompt",
"FL_Audio_Beat_Visualizer": "FL Audio Beat Visualizer",
+21
View File
@@ -0,0 +1,21 @@
# Timeline references and storyboards
All storyboards requested in one Writer response are submitted in one graph of independent asynchronous Nano nodes, without a local concurrency-count cap. Queue items still execute normally; provider throttling, request-size validation and shared batch completion affect actual speed. Large requests may spend credits simultaneously. There are no automatic paid retries.
Timeline and moodboard images use local 256-pixel WebP thumbnails with cache revalidation. Originals load only in the image popup, alongside available generation metadata. Reroll/remove controls sit below each timeline image. Loading indicators distinguish thumbnail loading from queued/generating jobs; reduced-motion preferences are respected. Wheel gestures over reference cards zoom/pan the timeline normally.
All sections requested in a Writer response share the same continuity brief and snapshot of checked moodboards. For stronger consistency, supply a clear character sheet and a style reference with explicit roles. Text alone cannot guarantee identical designs. Rerolls reuse the original brief and moodboard references; ask the Writer for new storyboards to use changed moodboard selections. A first generated master reference can be uploaded as a checked moodboard for subsequent sections; automatic master-reference generation is not implemented.
Ask Beat Writer to generate storyboards for the selected sections. It writes chronological image prompts and queues official Nano Banana 2 Partner Node jobs in ComfyUI. Prefer 2×2 grids for sections up to six seconds, 3×3 for longer sections.
Completed sheets are split into panels locally and assigned automatically. Active sheet thumbnails on timeline clips represent selected video references. If a section changes or attachment fails, its saved sheet remains visible as **not attached**, with a reason and an **Attach** button that uses no generation credits. **×** dismisses an unattached result without changing active references. Generated storyboards take precedence over reference assignments for the same section in one Writer response. Click a thumbnail to preview; **↻** generates a replacement. The old references remain active while a reroll runs. Generation and rerolls use ComfyUI credits without another approval dialog.
Four moodboard slots accept uploads, drops and pasted images. Add an optional role and check **Use** to send that image to the Writer and Google's image-generation endpoint. Unchecked images stay local. Checked moodboards and chat attachments share the Writer's eight-image limit.
The scheduler must connect directly to **FL MiniMax H3 Beat Shot Planner**. Storyboard generation connects **FL Prompt Reference Library** automatically and sets the planner's visual reference mode to **full**. Custom section references replace the planner's default media. The timeline soundtrack is unchanged. Panels are visual references, not exact timed keyframe constraints.
The optional **Section references** inspector supports manual image, video and audio assignments. `<Picture N>` and `<Video N>` each start at 1 within their media type, in selection order. Video audio remains paired; audio numbering also includes the timeline song. Only selected files are loaded and encoded. Grouped render sections must share the same ordered references.
Leave the editor open while the Writer prepares image requests. Submitted image jobs continue in ComfyUI if the editor closes; reopening recovers their results. Edited or deleted sections do not receive stale results automatically. Reconnecting cannot repeat an already submitted paid request. Check ComfyUI history for failed or unknown submissions before generating again; cancellation does not guarantee a refund.
Workflows store metadata, not embedded media. Share the referenced files under `input/fl-prompt-references`, `input/fl-beat-writer` and `output/fl-storyboards` with the workflow, preserving relative paths. Job records stay in the local Beat Writer data directory. Credentials are not stored in these manifests.
+15
View File
@@ -0,0 +1,15 @@
# Dynamic Scan FX analysis
Reference-file change detection belongs to the scheduler's serialized reference settings. The reference library no longer fingerprints its connected schedule: ComfyUI does not supply linked values during fingerprinting, which previously raised a warning and invalidated downstream caches on every run. Selected reference-file path, modification time and size still invalidate the scheduler when changed.
Connect a video batch and its FL prompt schedule to **FL Scan Video Shots**. Its IMAGE list runs one shared depth/normals/detection/pose branch for every authored shot. Adjacent sections with the same render group stay together. Without a schedule the complete video is one shot. A supplied schedule must match the video's frame count and cover it without gaps or overlaps.
Package each result with **FL Scan Analysis**, then connect **FL Scan Analysis Collect** to one Interactive Scan FX analysis socket. The collector preserves order and shot boundaries without concatenating analysis tensors. Existing manually connected analysis sockets still work. Processing follows actual shot boundaries, not arbitrary quarters; this changes effect resets compared with the old four-quarter layout.
For VFX-only iteration, use an **FL Switch** with the saved raw video on `on_false` and the H3 assembler on `on_true`. Set it to false, and disable any separate output nodes that still request the H3 branch. The saved video is an explicit input, not a cache that automatically updates when prompts change. Select a new saved take after generating one; retain its matching schedule and FPS.
ComfyUI's RAM-pressure cache can evict analysis results. On builds with active/inactive headroom arguments, `--cache-ram 10 16` retains results when memory permits while keeping headroom. These are free-RAM thresholds in GB, not cache-size limits. Keep the flags in the launcher used for subsequent starts. A saved source prevents H3 reruns even after cache eviction or restart; analysis may still rerun.
The synthetic Scan FX demo renders at 320 pixels and up to 12 FPS, applies color filters once per surface, and stops animation scheduling when paused, offscreen, inactive, or in a hidden tab. These settings do not alter final video rendering.
Voxel polygon coordinates and face colors are prepared in arrays before rasterization, preserving painter order and OpenCV drawing. Saturation and edge glow use ComfyUI's selected compute device one frame at a time, including transfers back to CPU output. Low GPU-memory headroom selects CPU instead. Brightness-mask preparation, projection, optical flow and cursor composition remain CPU-based. GPU arithmetic may differ from CPU by floating-point roundoff; this is not a fully GPU-rendered compositor.
+25
View File
@@ -0,0 +1,25 @@
# Dynamic parallax posters
`FL Image Layer Planner` (existing node ID `FL_PosterLayerPlanner`) inspects one finished image using a vision-capable CLIP, or accepts a manual JSON layer list. Auto mode chooses up to the requested budget, including one filled background. Grouping controls whether related objects stay together; `artwork_type` guides photography, illustration, graphic design, products and mixed media. Text layers are only planned when lettering is visible. Auto can correct an invalid plan once and removes other-object location clauses from cutout targets; Manual prompts remain unchanged. Typography is extracted from the generated artwork, not rendered as an overlay.
Manual plans use this form:
```json
[
{"name":"Paper", "kind":"background", "prompt":"Cream paper without objects or lettering", "depth":12},
{"name":"Headline", "kind":"text", "prompt":"The large black AFTER HOURS lettering at the top", "depth":8},
{"name":"Hero", "kind":"art", "prompt":"Woman wearing a silver jacket and headphones", "depth":4}
]
```
`FL Poster Layers` expands the plan into native ComfyUI conditioning, sampling, and decode nodes. Use **Qwen Image Layered Control**, its text encoder, and the layered RGBA VAE. The image reference is encoded once; the Control latent has `layers=0`, meaning one RGBA result per extraction. Each layer has its own stable sampler ID and seed.
The review panel shows small thumbnails and full-resolution RGBA on click. Edit a basic-English extraction prompt, depth, scale, offset or visibility, then Run. Reroll changes one layer's seed and queues the workflow. Reset removes that layer's edits for the next Run. Edits are scoped to the plan; changed plans do not inherit stale object prompts.
Connect the resulting `FL_PARALLAX_STACK` to **FL Layered Parallax**. It supplies the background and a variable-length list of cutouts. Existing individual layer inputs still work. Lower depths are nearer and move faster; keep the camera background depth above every cutout depth.
The compositor now includes [Parallax Studio](parallax_editor.md), a visual camera editor with live low-resolution previews, presets, drag controls and cover/contain framing. Width, height and frames remain standard widgets.
Layout changes only affect stack assembly/composition. Prompt edits or rerolls invalidate the affected extraction path, not every sampler. A node-scoped cache provider recovers exact RGBA tensors when ComfyUI's RAM cache evicts them. It uses ComfyUI's native dependency signatures, writes safetensors atomically under `user/__cache/fl_poster_layers_v1`, and keeps at most 1 GB by removing its oldest cache files. No large tensor store is retained by the provider. Clearing that folder forces fresh extraction; exported PNGs remain untouched. RGBA assets and 192-pixel thumbnails are saved under `output/Dynamic_Parallax_Poster/layers` and `thumbnails`, so saved reviews survive a server restart.
These models generate their extraction: lettering can change, small items can be omitted, and hidden regions can be invented. Check the source spelling, alpha previews, and reassembled still before exporting motion. Auto planning is not a guarantee of perfect semantic separation.
+89
View File
@@ -0,0 +1,89 @@
# H3 Street Scan
A local style study inspired by [Ingi Erlingsson's “please hold”](https://x.com/ingi_erlingsson/status/2097056649559134367), using newly generated footage rather than frames from the reference.
## Audio-reactive Chaos variant
### Native depth camera and audio mapping
Interactive Scan FX includes our own CPU depth-parallax renderer; it does not depend on Depthflow, OpenGL, or another model. It extends the existing depth projection with horizontal/vertical camera offsets and dolly, counter-adjusted around a selected steady-depth plane. `current` preserves the original renderer. `depth_parallax` can affect the `whole_scene` or only the voxel/depth layers inside reveals (`reveals_only`). Both remain 2.5D: newly exposed surfaces are filled from visible samples, not reconstructed.
Reveal weights choose voxel normals, edge scan, or depth **once per gesture** using the seed. Zero disables a layer; the total must remain positive. Depth styles are grayscale, false color, and contours. Equal voxel/edge weights with depth disabled preserve the original selection. Cursor activity thins gestures; reveal size is sampled at gesture start.
The on-node Audio Mapping panel assigns Kick/Snare/Hat envelopes to supported parameters over frame ranges `[start, end)`. Click a mapping to select its range, drag either timeline edge to resize, or drag its interior to move it. Clicking outside the selected range seeks the preview. Numeric range fields are available too. Overlapping enabled mappings for the same parameter are rejected. Mappings replace the parameter's value inside the range; outside it the base value remains. Brightness, saturation and glow retain their original envelope reactions outside mapped ranges, so mappings do not double-apply them.
Smoothing is optional, measured in seconds, and resets at shot boundaries. Speed mappings accumulate animation phase without resetting it. Cube size, FPS and seed are not modulation targets. The timeline shows the last rendered values and authored shot boundaries; settings changes require another execution. A Depth diagnostic tab shows the projected map before camera accents. Old workflows retain the original defaults until the new options are enabled.
The current workflow consolidates the effect into **FL Interactive Scan FX**. Each **FL Scan Analysis** packages a shot's depth, normals and optional masks/detections/pose; connect these in chronological order to the growing shot inputs. Their total frame count and resolution must match the source video, and all three envelopes must match its FPS and frame count. Generation and analysis remain external and cacheable.
The unified node reuses voxel rendering, digital depth projection, audio-locked cursor editing, masked snare brightness, kick saturation and kick edge glow. Its main widgets control the surface, cursor count, camera motion and seed. The Advanced panel contains projection, cut timing, cursor scale/reveal strength and finishing controls. Existing individual nodes remain available.
Its silent diagnostic preview has **Final**, **Surface**, **Mask**, and **Debug** tabs, a frame scrubber and synchronized Kick/Snare/Hat meters. Surface shows projection before camera accents; Debug overlays planned gesture trajectories on the final result. Previews describe the last execution, not live parameter changes. They are temporary MP4 files; rerun if ComfyUI clears its temp directory. Full-resolution final/surface/mask outputs and a timing report remain available. The labeled FL Video Combine output retains audio.
The current Y2K workflow uses `digital_layers` on all four Scan Composite nodes: a full-scene depth projection with filled splat gaps, crisp white borders and offset cobalt/black backing panels. It does not use the eroded fragment mask or temporal edge ghosts. `fragment` remains the default for older workflows; raggedness and echo controls apply only to that mode.
Frame- and second-based prompt schedules can run with no detected beats. Beat-index schedules still require a beat grid. The current workflow uses scheduler envelopes 1/2/3 as kick/snare/hat controls: every beat, every other beat, and short sixteenth-note pulses. These are authored grid events, not detected instrument stems. Its four 48-frame sections produce 192 frames (8 seconds), using seed 65, 10 SA Solver steps, beta57, CFG 1 and denoise 1. Earlier test settings below describe the original Chaos version.
**FL Voxel Normal Relief** now sits between Depth to Normals and each Scan Composite. It draws normal-colored cuboids with shaded sides; relative depth sets their height and a seeded smooth wave adds subtle animation. The four shared controls are cube size (12 px), relief (0.65), animation (0.18), and speed (0.7 cycles/second). This is a 2.5D image effect, not a voxel mesh. Keep its depth and normals aligned, and connect it before projection so the cursor reveals follow the same scene and camera transforms. The raw normal previews remain unchanged. FL Video Combine saves the labeled before/after with audio.
The primary **H3 Street Scan Chaos - Generate and Composite** workflow uses **FL Audio Beat Prompt Schedule** as its audio source, frame-accurate trim and prompt timeline. The test crop is `Inspired by dnb.mp3`, frames 768–1032 at 24 fps (32–43 seconds). Its locally installed Beat This model supplies beat timing.
The schedule feeds **FL MiniMax H3 Beat Shot Planner**, including its matching audio and reactive prompt-envelope outputs. Each render receives a shot-local audio reference and rebased beat-weighted motion prompts. **Beat KSampler** uses the Turbo LoRA, four Euler steps, simple scheduling and the existing 12/3 video/audio sigma shifts. **Shot Assembler** removes H3 padding. Audio conditioning encourages musical movement; it does not guarantee frame-perfect physical accents.
**FL Scan Audio Edit** defaults to `audio_locked` in this workflow. It preserves every source frame chronologically and adds abrupt camera cuts, punch-ins and tilt without retiming generated motion. Its backwards-compatible `remix` mode still permits random seeks and playback-rate changes; those intentionally discard diffusion-time alignment. Keep the four analysis slices and shot-length settings consistent with the authored prompt sections when changing section boundaries.
Cursor gestures start independently on audio accents rather than restarting at every cut. A fixed seed controls their positions, directions, curved paths, sizes, durations and normal/edge layer choices. Silence produces no new gestures. The report includes cursor events and exact source-frame mapping. The reveal mask excludes cursor graphics and can drive downstream masked effects.
The existing Fill reactive brightness, saturation and edge-glow nodes provide additional modulation. The current workflow no longer uses Drum Detector.
The older Chaos FX Lab remains a cached-footage remix workflow. The primary workflow no longer depends on those clips. All Chaos outputs are saved under `output/FL_H3_StreetScan_Chaos`.
## Workflows
- **H3 Street Scan - Generate and Composite**: four editable H3 prompts, native Turbo sampling, analysis, effects and assembly.
- **H3 Street Scan - FX Lab**: the same effects applied to saved H3 clips. Use this for fast look development or replace the four video paths with your own footage.
Both are saved in `user/default/workflows`. Rendered videos and workflow copies are in `output/FL_H3_StreetScan`.
Four 66-frame sections at 24 fps produce an 11-second, 640×640 silent montage. The comparison output places the original on the left and the scan treatment on the right. FX Lab paths refer to local generated clips; include those clips and update paths when sharing. The full generation workflow does not depend on those cached files.
## Pipeline
H3 generates full-frame footage. Depth Anything V2 estimates relative depth; blurred depth feeds Depth to Normals. Impact/Ultralytics supplies person segmentation and face/hand detections. DWPose supplies body and hand joints. `FL Scan Video Detections` matches box IDs between adjacent frames. `FL Street Scan Composite` applies depth reprojection, a chipped scene boundary, normal-map flashes, tracking graphics, pose flashes and short echoes.
This is **2.5D reprojection**, not a multi-view reconstruction or exportable scene mesh. Depth annotations are normalized relative estimates, not metric coordinates. Tracking IDs are nearest-neighbor associations, not persistent identity recognition. The compositor accepts DWPose's pixel-coordinate `POSE_KEYPOINT` batches; pose input is optional.
Each shot is analyzed separately so depth smoothing, feature tracking and echo history reset at cuts. Source, depth, normals, masks, detections and pose must describe the same frames and resolution (pose includes its own canvas dimensions).
## Controls
- **Orbit / depth relief:** parallax. Larger values expose more missing surfaces and reprojection gaps.
- **Scene scale / raggedness:** framing and chipped boundary. The detected subject is retained before reprojection.
- **Normal mix:** surface-color flashes and the rectangular normal-field panel.
- **HUD / pose opacity:** detection graphics and body/hand skeleton flashes.
- **Echo strength:** two-frame edge trails.
Seeds are fixed for repeatable tuning. Depth and normal previews are below each analysis branch. Changing an effect control does not require another H3 generation; Comfy's execution cache can also reuse unchanged analysis while it remains available.
## Live effect preview
The shared workflow's four analysis branches use **FL Scan Video Section**, indices 0–3 with `sections=4`, all connected to the same video as Interactive Scan FX. Boundaries are derived from the input frame count, so the chunks cover the complete video exactly once, including uneven remainders. These are processing chunks, not detected scene cuts. Keep all four branches connected; video/envelope FPS and duration must still agree.
Minimum/maximum cut-frame controls follow each other when crossed. Reversed ranges in older workflows or API requests are sorted before rendering; they do not abort the effect.
The right-hand editor uses Voxels, Camera, Reveals, Overlay and Finish tabs. Numeric controls have bounded sliders/fields; `?` opens parameter help. Reset restores the base value without removing mappings. Tab reset requires confirmation. Solo changes the demo view explicitly; switching tabs does not.
Drag a colored Kick/Snare/Hat chip onto a highlighted parameter, or click the chip then the parameter (keyboard: focus the card and press Enter). Click its colored badge for range, smoothing, invert and enable controls. Existing assignments offer replacement or a new unused time range. Overlapping enabled ranges are rejected in the editor and backend. Reset mapping preserves its assigned time section; Remove mapping disconnects it. The timeline below the preview edits the selected badge's range; rendered-value meters show the last execution, not live demo predictions.
Interactive Scan FX has a two-column editor: visuals on the left and effect controls on the right. **Live demo** is a lightweight synthetic depth scene available before execution; use All, Voxels, Depth, Reveals, Overlay, or Finish to inspect approximate effects. Pause the demo to compare parameter changes at a fixed time. It does not run models, queue work, or predict the exact output. Audio mappings, detection, occlusion and edit timing still require a render. **Last render** retains the actual diagnostic video and envelope timeline. Connected parameter inputs take precedence over stored controls; the demo uses stored values. Changes still require execution to update output videos.
## Installed dependencies used
- ComfyUI native MiniMax H3 and FL MiniMax H3 LoRA Block Loader.
- H3 `minimax_h3_ref2va_pruned_int8_convrot.safetensors`, Qwen3-VL H3 NVFP4 text encoder, H3 video/audio VAEs, and the ref2v Turbo 4-step LoRA.
- DepthAnythingV2: `depth_anything_v2_vitl_fp32.safetensors` (bf16 compute).
- Impact Pack/Subpack: `person_yolov8m-seg.pt`, `face_yolov8m.pt`, `hand_yolov8s.pt`.
- ControlNet Aux DWPose: `yolox_l.torchscript.pt`, `dw-ll_ucoco_384_bs5.torchscript.pt`.
- Image Filters (Depth to Normals), KJNodes (side-by-side concatenation), VideoHelperSuite, and Fill Nodes.
No paid generation API or new model download was used for this test. The named third-party loader nodes may download missing weights on another installation.
+115
View File
@@ -0,0 +1,115 @@
# FL Krea Reference Control — proposed scope
Status: implemented as FL Krea Reference and FL Krea Reference Guider. See [usage](krea-reference-control.md). Visual validation results are recorded separately; the initial scope below describes the design and acceptance targets.
Implementation finding: independent full-image prediction blending still copied source portraits and panel layouts in the three-seed comparison. The delivered node therefore adds Context mode (default), which retains image-informed text states after the vision block, and Full mode, which retains visual tokens as well. Both retain all twelve layers. Context removed the unwanted portraits/panels in that test set; this is not a universal guarantee.
## Goal
Generate one coherent image from a scene prompt and independently controlled references. Let the user request the linework from one image, the palette from another, and the subject or composition from another, with adjustable contribution and timing.
Initial target: the installed Krea 2 Turbo model, Qwen3-VL-4B Krea encoder, and the user's 8-step Euler/simple workflow. Role instructions express intent; they are not trained, disentangled style/identity controls.
## Findings driving the design
- Rebalance's `compile_edit` serializes multiple images into one Qwen conversation as `Picture 1`, `Picture 2`, etc. It does not build a pixel grid or independently mix image contributions. A generated grid is a possible model interpretation, not an explicit image assembly operation in this code.
- `Krea2EditRebalance` retains only one of twelve conditioning taps. Its complementary reference branch has no effect under the current per-band projection math. This was reproduced with synthetic tensors.
- Its private pass cache omits image content and CLIP identity. Different same-size images can return identical cached conditioning. This was also reproduced.
- Both existing nodes use vision-language conditioning, without reference latents. The new node must not advertise exact identity, spatial, or pixel preservation on that basis.
- Krea already uses ComfyUI's optimized attention. A custom guider can combine model predictions using existing sampling interfaces without replacing attention implementations or changing core model code.
## User-facing nodes
Two node types keep individual reference settings beside each image while keeping the sampling controls centralized.
### FL Krea Reference
Produces an `FL_KREA_REFERENCE` descriptor containing the image and its interpretation settings. It performs no model encoding itself.
| Input | Meaning |
| --- | --- |
| image | One reference image. A batch with more than one image raises a clear error; use existing batch selection/splitting nodes. |
| enabled | Exclude this reference completely when disabled. |
| role | Style, palette, subject, composition, or custom. Default: style. |
| instruction | Optional description of what to borrow and what to leave behind. Editable text supplements the role. |
| weight | Nonnegative direct strength of the reference contribution. Zero excludes the reference. Initial neutral value: 1. |
| resolution | Longest-side limit: 256, 512, 1024, or 1280. Initial default: 512; preserve aspect ratio. |
| start / end | Advanced activation window in diffusion progress, from 0 to 1. Default: full run. |
| fade | Advanced fade fraction at each edge of the activation window; capped to prevent overlapping ramps. Default: no fade. |
No arbitrary four-reference limit. Practical cost is shown in documentation. Initial performance validation covers one, two, and four references.
Roles generate explicit instructions, not hidden numerical layer presets. For example, style requests medium, linework, shading, and texture while the scene prompt defines the subject. Composition requests broad arrangement without claiming pose locking. Custom uses the user's instruction directly.
Source cropping uses existing image nodes. Output-region masks and automatic subject extraction are outside the first version.
### FL Krea Reference Guider
Inputs: MODEL, CLIP, scene prompt, an optional `io.Autogrow` collection of reference descriptors with minimum zero, and overall reference influence.
Output: GUIDER, for the existing `SamplerCustomAdvanced` node. Noise seed, sampler, sigma schedule, latent size, and VAE decode stay in standard ComfyUI nodes. No model or image pass-through outputs.
Overall influence initially spans 0 to 1, with a provisional default of 0.7. Zero is the plain text baseline; one applies the entered reference weights wherever references are fully active. Combined contributions above one extrapolate beyond the baseline.
The example workflow replaces KSampler with RandomNoise, KSamplerSelect, BasicScheduler, and SamplerCustomAdvanced, retaining Euler/simple, eight steps, fixed seed, and existing latent/decode nodes. This is an intentional sampling interface change; a CONDITIONING-only output cannot provide this independent prediction blend to a stock KSampler.
## Proposed sampling behavior
1. Encode the scene prompt as a text-only baseline using Krea's native template.
2. Independently encode the same scene prompt with each enabled reference and its role instruction. Each reference branch sees only one image. Preserve all twelve conditioning taps and their metadata.
3. At each model evaluation, calculate the baseline prediction and each active reference prediction at the same latent and sigma.
4. Combine predictions in their shared latent coordinates. Do not average text or visual embeddings by token index: different sequences do not have reliable token correspondence.
Let `D0` be the baseline prediction, `Di` each reference prediction, `S` overall influence, `wi` direct enabled weights, and `ei(t)` the reference activation envelope:
`D = D0 + S * sum(wi * ei(t) * (Di - D0))`
This is a blend of denoiser predictions, not an image crossfade or a blend of random seeds. The whole reference-conditioned branch contributes, including its role instruction. It is not a mathematically isolated image-only effect.
Apply positive enabled weights directly, without normalization. Fading one reference therefore reduces its contribution without automatically boosting another. With no references, all weights zero, or S=0, skip reference encoding/evaluation and follow the ordinary text-only CFG-1 path. Skip baseline evaluation when its coefficient is exactly zero. Disabled/zero-weight entries never influence other weights or prompts.
Schedules use ComfyUI's model sampling conversion from progress boundaries to sigma. Interpolate fades between those converted boundaries, documenting that interpolation occurs in sigma space. Do not count Python callbacks as diffusion steps. Fractional denoise and samplers with multiple evaluations must follow the same sigma-based contract.
Independent branches remove the shared multi-picture input as a source of panel layout. They cannot guarantee that no grid will be generated, especially when a reference itself contains panels. The effect of weighting on visual style is not guaranteed to be linear or monotonic.
## Implementation boundaries and cost
- Add one Python module under `nodes/conditioning/`, register two nodes through the pack's existing mappings, and add focused tests plus an example workflow. Reuse the installed V3 schema patterns for Autogrow.
- Use the existing CFGGuider lifecycle and `calc_cond_batch` contract. Register every conditioning branch through the guider so ComfyUI owns hook preparation, devices, loading, offloading, and cleanup.
- Evaluate branches sequentially initially and accumulate predictions to avoid retaining N full prediction tensors. No custom attention kernels, backend selection, raw model mutation, extra model copies, dependencies, downloads, or internet access.
- Keep the twelve-tap encoder layout intact. Validate the supported Krea model/encoder combination at the node boundary, with an actionable mismatch error.
- Scope is CFG-1 Krea Turbo. Additional negative-prompt guidance, other Krea variants, and interactions with guidance-modifying third-party patches need separate validation. Preserve supported standard hooks and report unsupported combinations explicitly.
- Worst-case denoiser work is N+1 evaluations per sampler evaluation: four references plus the baseline can cost approximately five evaluations. Wall-clock time and VRAM must be measured, not inferred from this count alone.
- No private persistent tensor cache. Ordinary ComfyUI graph caching applies; changing settings that invalidate the guider can initially re-encode its references. Future encoding reuse requires a separate measured need and correct ownership/invalidation.
- The descriptor and guider outputs may hold their execution data under normal graph caching. Do not introduce an additional store that retains images, embeddings, or predictions across executions.
## Prototype and acceptance gates
First establish whether independent prediction blending visibly improves reference mixing. Do this before polishing controls or adding presets.
Use a small fixed-seed comparison set: the current dog/illustration case, two clearly different visual styles, palette plus style, subject plus style, and references that contain panels. Compare native text-only, current joint Encode Rebalance, and the proposed blend, holding prompt, seed, size, model, and sampler constant. Use at least three seeds for each selected comparison.
Review prompt adherence, contribution from both references, unwanted panels/duplicates, source-subject leakage, recognizable requested style features, artifacts, latency, and peak VRAM. Record comparisons for user review; do not label a subjective quality improvement as proven from synthetic tests.
Required numerical and integration checks:
- No references and S=0 match the native text-only baseline within the established numerical tolerance.
- One reference at S=1 with a full envelope matches that branch alone; the baseline can be skipped.
- Removing a zero-weight or disabled reference leaves the result unchanged.
- Swapping same-size image contents invalidates upstream execution and changes encoded conditioning when the images encode differently; changing CLIP invalidates encoding as well.
- Reordering complete reference descriptors preserves the mixture within floating-point tolerance.
- Doubling weights doubles the reference difference from the baseline at a fixed latent and sigma. A lone reference at 0.05 contributes 5% as much as at 1, before overall influence.
- Weights, schedules, and short/fractional-denoise runs follow the documented formula. Validate zero-width windows and out-of-range inputs at the boundary.
- Multiple differently sized images and prompts with different sequence lengths work without token-index averaging or padding-based correspondence assumptions.
- Repeated execution, cancellation, and sampler failure leave no additional model-owned cache or patch behind.
- Existing optimized attention, model patches, supported dtype/device handling, and low-VRAM execution continue through ComfyUI's lifecycle. Run available hardware checks; explicitly record untested backends.
If the blend produces washed-out styles, weak identities, or persistent competing compositions, revisit the conditioning strategy before release. Do not compensate with unexplained layer multipliers or promise a grid-prevention switch.
## Deferred work
Per-region destination masks, trained identity preservation, latent-reference control, automatic image captioning, per-layer expert knobs, extrapolating reference strength, a custom canvas editor, and a faster attention-based mode are outside v1. Each needs its own evidence and implementation contract.
## Deliverables
Two registered nodes, focused regression tests, a concise usage guide, an example replacing the user's current sampler path, and fixed-seed visual comparisons with timing/VRAM measurements. No existing Rebalance code or user workflow is modified as part of this scope document.
+41
View File
@@ -0,0 +1,41 @@
# Krea reference control
Use **FL Krea Reference** for each image and connect its output to **FL Krea Reference Guider**. Connect the guider to **SamplerCustomAdvanced**, with standard noise, sampler, scheduler, and latent nodes. The example uses Krea 2 Turbo, eight steps, Euler/simple, and a fixed seed.
For **KSampler** or **KSampler Advanced**, connect the guider node's **MODEL** output to `model` and **CONDITIONING** output to `positive`. Both connections are required to preserve independent reference weights and timing. Connect an ordinary empty CLIP Text Encode to `negative`. Use CFG **1** to match the GUIDER output; higher CFG applies normal negative guidance after the reference blend. The original GUIDER output remains available. This changes sampler compatibility, not the reference blending method or its potential for overlapping subjects.
The guider encodes each reference separately, preserves all twelve Krea conditioning layers, and blends model predictions during sampling. It does not place the reference images into a shared picture grid.
## Controls
- **Role:** select what to borrow. Style targets medium and texture; palette targets colors; subject targets appearance; composition targets arrangement. Custom uses only the main scene prompt. These are model instructions, not guaranteed identity or geometry controls.
- **Weight:** direct reference strength. 0.05 applies 5% of that reference?s contribution before overall influence; 1 applies the full contribution. Zero or disabled excludes it. Weights are not normalized against other references.
- **Influence:** overall reference strength after blending. Zero gives the plain scene prompt. Start at 0.7 and compare using the same seed.
- **Blend mode / average amount:** Add preserves the additive blend. In Average mode, amount 0 is additive and amount 1 divides contributions by the number of enabled references; intermediate amounts blend those results. Four enabled references at weight 1 contribute 25% each at full averaging. Individual weights still apply. Disabled references are excluded from the count; zero-weight or faded references retain their share, which returns to the scene prompt. Existing workflows default to Add.
- **Resolution:** reference longest-side limit. Start at 512; use 1024 for fine detail. Larger settings increase encoding cost. Images are not enlarged except to meet the minimum vision patch dimensions.
- **Reference mode:** Context (default) lets Qwen see the image, then supplies only image-informed text states to Krea. This reduces direct copying of the source layout. Full supplies visual tokens as well, for stronger source resemblance. Both preserve all twelve conditioning layers.
- **Start / end / fade:** advanced timing controls in full diffusion progress. Start must be below end. Fade is a fraction of the window at each edge, up to 0.5. Fade interpolation occurs between converted sigma boundaries. Shortened denoise runs use the corresponding part of this full schedule.
Reference sockets grow as you connect them, up to ComfyUI's 100-entry Autogrow ceiling. Each reference accepts one RGB image; select a frame first when the source is an image batch.
For a dog rendered like an ink illustration, describe the scene in the main prompt and select a style reference. A second palette reference can supply its colors independently.
## Cost and behavior
Each active reference normally adds one model evaluation per sampling evaluation. Two references plus the text baseline require three evaluations. The baseline evaluation is skipped when the combined active reference contribution is exactly 1. Timed-out branches are skipped. This is slower than encoding all pictures together; it makes their numerical contributions independent.
Weights directly scale each reference?s difference from the scene baseline. Fading one reference returns its contribution to the scene baseline without boosting the other references. The blend is:
`baseline + influence * blend_scale * sum(weight * envelope * (reference_prediction - baseline))`
`blend_scale` is 1 in Add mode. In Average mode it is `1 - average_amount + average_amount / N`, where `N` is the number of enabled references. No enabled references uses the baseline. Both GUIDER and MODEL/CONDITIONING outputs use this blend.
When active weights multiplied by influence sum above 1, the guider extrapolates beyond the baseline instead of silently normalizing or clipping the sliders. Lower overall influence or the reference weights for a gentler result.
The visual result is nonlinear. A larger reference weight does not guarantee proportionally stronger style. References containing panels can still suggest panels, especially in Full mode, and unrelated subject references can compete. Compare a reference alone before mixing it with others. Context can weaken source identity and geometry; use Full when those matter more than avoiding copied layouts.
The nodes use normal ComfyUI execution caching, with no private image/embedding cache. Changes to reference settings can re-encode references. Changing same-size image contents or the text encoder is not hidden by a shape-only cache.
Use a Krea 2 model and CLIPLoader type `krea2`. The guider uses CFG 1 and rejects custom CFG/pre-CFG/post-CFG/batch-guidance functions. Ordinary model patches and optimized attention remain managed by ComfyUI. Other guidance or cross-call caching patches have not been validated with independent reference branches.
This version does not supply reference latents, destination masks, identity training, or automatic captioning. Use existing image nodes to crop a source reference.
+45
View File
@@ -0,0 +1,45 @@
# Krea reference validation
Tested locally on 2026-09-11 with Krea 2 Turbo FP8 scaled, Qwen3-VL-4B FP8 scaled, Qwen Image VAE, and an NVIDIA RTX PRO 6000 Blackwell Max-Q. Eight Euler/simple steps, CFG 1.
## Automated checks
17 focused unittest cases pass. They cover the weighted prediction formula, zero influence, the single-reference endpoint, reference order and weight scaling, disabled/zero-weight exclusion, repeat sigma evaluations, fade boundaries, schedule independence, input validation, image/CLIP re-encoding, independent image encoding, twelve-layer preservation, Context suffix copying/mask slicing, and schema/model validation.
The tests use synthetic predictions for numerical invariants. Real server runs additionally exercised the V3 Autogrow inputs, actual model loading, vision encoding, denoising, and VAE decoding.
The weight-slider regression tests now check a single reference at 0, 0.05, 0.25, 0.5, 1, and 1.5; independent removal; and combined weights above one. Weights directly scale the baseline difference. Earlier versions normalized weights, making a lone reference effectively on/off.
After the correction, a live-server sweep rendered a single reference at weights 0, 0.05, 0.25, 0.5, and 1 in both Context and Full modes, with a fixed seed and overall influence 0.7. All five decoded RGB outputs were distinct in each mode. Mean absolute RGB differences from weight zero were 0, 10.92, 28.77, 39.35, and 59.59 for Context, and 0, 12.34, 27.55, 50.15, and 85.53 for Full, on the 0–255 channel scale. The inspected low-weight images stayed close to the baseline; higher Full weights increasingly copied source faces/panels. This demonstrates working numerical strength control in the test case, not universally linear visual response. Images are in `output/Krea_Weight_Test/`.
## Initial fixed-seed visual comparison (before the weight correction)
The same dog-in-a-park prompt was generated at 512x768 with seeds 193328989555475, 193328989555476, and 193328989555477. References were a monochrome portrait with window panels and a blue/violet illustration.
Compared native text-only generation, zero reference influence, each reference alone, independent Full-mode blending, the old joint Encode Rebalance node, and the new Context-mode paths. Zero influence produced pixel-identical decoded RGB images to the native baseline for all three seeds.
The joint encoder copied portraits and panels in all three comparisons. Independent Full-mode blending also retained unwanted source faces and windows. Context-mode style and mixed outputs preserved the dog scene without those unwanted portraits/panels in the inspected set. The palette-only branch visibly changed the color treatment. With style weight 1 and palette weight 0.5, the blend remained closer to the style branch; relative weights are not linear measures of perceptual influence.
These are observations from a small test set, not evidence of universal style/identity separation or grid prevention. Subject and composition roles require further use-case validation.
## Timing and memory
Warm server execution times at 512x768, excluding queue wait:
| Path | Time |
| --- | --- |
| Text-only / zero influence | About 1.7 seconds |
| One Context reference | 3.3–3.6 seconds |
| Two Context references | About 5.0 seconds |
| Four Context references | About 8.4 seconds |
The first Context run incurred model-loading overhead and is excluded from warm timing. A four-reference run and a faded reference window with denoise 0.75 also completed successfully.
The updated live Krea workflow also completed at 1024x1536 in 28.35 seconds of server execution, using the user's current woman-on-a-park-bench prompt and two current illustration references. The result followed the scene without constructing a reference grid. Saved locally as `output/Krea_Reference_Test/live_workflow_00001_.png`.
Polling ComfyUI system stats during the warm Context runs observed about 33.7 GiB total device memory in use. This includes resident models and other GPU allocations; it is not incremental per-reference VRAM or an exact peak. The dynamic allocator is not fully represented by PyTorch's reserved-memory counter.
CUDA was exercised. CPU, ROCm, MPS, DirectML, XPU, NPU, explicit low-VRAM mode, cancellation during sampling, and arbitrary third-party patches were not separately validated. The implementation uses ComfyUI's existing loading, offloading, attention, and guider cleanup paths.
## Standard KSampler outputs
The guider now also outputs a patched MODEL and CONDITIONING for standard samplers. All 20 unit tests pass, including parity for independent weights, reference schedules, CFG, plain conditioning, and preservation of the original model. A real 512x768, eight-step Euler/simple KSampler render with two references and fading matched SamplerCustomAdvanced pixel-for-pixel at CFG 1 and the same seed. This verifies sampler parity, not improved reference composition quality.
+147
View File
@@ -0,0 +1,147 @@
{
"last_node_id": 3,
"last_link_id": 2,
"nodes": [
{
"id": 1,
"type": "UNETLoader",
"pos": [
60,
80
],
"size": [
440,
86
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
1
],
"slot_index": 0
}
],
"title": "Fine-tune: Dirtyrealism v5",
"properties": {
"Node name for S&R": "UNETLoader"
},
"widgets_values": [
"Krea\\krea2SATDirtyrealism_v5_fp8.safetensors",
"default"
]
},
{
"id": 2,
"type": "UNETLoader",
"pos": [
60,
280
],
"size": [
440,
86
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "MODEL",
"type": "MODEL",
"links": [
2
],
"slot_index": 0
}
],
"title": "Base: Krea2 Turbo",
"properties": {
"Node name for S&R": "UNETLoader"
},
"widgets_values": [
"Krea\\krea2_turbo_fp8_scaled.safetensors",
"default"
]
},
{
"id": 3,
"type": "FL_ModelDifferenceLoraSave",
"pos": [
580,
130
],
"size": [
470,
210
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "finetuned_model",
"type": "MODEL",
"link": 1
},
{
"name": "base_model",
"type": "MODEL",
"link": 2
}
],
"outputs": [
{
"name": "lora_path",
"type": "STRING",
"links": null,
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "FL_ModelDifferenceLoraSave"
},
"widgets_values": [
"Krea2/Dirtyrealism_v5_minus_Krea2_Turbo_rank128",
128,
"auto"
]
}
],
"links": [
[
1,
1,
0,
3,
0,
"MODEL"
],
[
2,
2,
0,
3,
1,
"MODEL"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 1,
"offset": [
0,
0
]
}
},
"version": 0.4
}
+18
View File
@@ -0,0 +1,18 @@
# Layered parallax
`FL Parallax Layer` accepts RGBA video or RGB plus a foreground mask (white = opaque). It adds depth, scale, screen-relative offsets and opacity, and produces a single checkerboard matte preview. An explicit mask overrides embedded alpha.
`FL Layered Parallax` takes an opaque background and expandable layer inputs. It sorts cutouts far-to-near and projects them using camera translation and forward travel. Smaller depth values move faster; the background must be farther than every cutout. Rendering uses premultiplied-alpha bilinear sampling in four-frame chunks on ComfyUI's selected device, with a CPU option. Outputs are the moving composite, the same animation with the camera locked, and a middle-frame depth preview (near = bright).
Camera controls:
- `motion`: burst/settle, eased glide, sinusoidal loop, or locked.
- `travel_x/y`: camera excursion in output widths/heights at depth 1; projected displacement divides by depth.
- `push_in`: forward travel in depth units. Keep every plane in front of the camera.
- `overscan`: enlarges only the background to cover movement. Beyond the source boundary, the background edge is extended; hidden scenery is not generated.
All animated plates must have equal frame counts. Single images are held over the longest animation. Set `frames` to animate still-image plates for a specified duration; zero inherits the source frame count. An explicit length must match any animated inputs. There is no silent trimming or frame-rate conversion. Set FPS in the exporter.
This is multiplane 2.5D composition, not a reconstructed scene or novel-view synthesis. It cannot expose unseen sides of objects. Alpha quality and independent generated motion affect the result. Loop camera movement does not make independently generated plates loop seamlessly.
Changing compositor controls does not change upstream generation or matting inputs. ComfyUI may reuse those cached results while resident; cache survival across restarts or memory pressure is not guaranteed. Save raw plates and RGBA assets for later reuse.
+9
View File
@@ -0,0 +1,9 @@
# FL Model Difference to LoRA
Connect two **Load Diffusion Model** nodes: the fine-tune to `finetuned_model`, and its original base to `base_model`. Queue **FL Model Difference to LoRA** to extract `fine-tune - base` and save a numbered `.safetensors` file under the first configured LoRA directory. The `filename_prefix` can include a subfolder, such as `Krea2/Dirtyrealism_difference`.
Rank defaults to 128. Higher ranks preserve more of the weight difference and produce larger files. `auto` uses ComfyUI's selected device; choose `cpu` if the device lacks SVD support or available memory. Extraction processes one parameter at a time, honors weight patches and quantization scales, and leaves the input models unchanged. Biases and other non-matrix parameters are stored as full differences in ComfyUI's LoRA format. No CLIP extraction is performed.
Apply the exported file to the same base with **Load LoRA (Model Only)** at strength 1. Low-rank compression is approximate, and quantized source models include quantization differences. Logged retained weight energy measures numerical reconstruction, not image quality. Compare generations before replacing a fine-tune in production workflows.
The included [Krea2 workflow](krea2_difference_lora.json) selects `krea2SATDirtyrealism_v5_fp8` and `krea2_turbo_fp8_scaled`, saving into `loras/Krea2`. Restart ComfyUI after installing the node, then open the workflow.
+25
View File
@@ -0,0 +1,25 @@
# Parallax Studio
FL Layered Parallax keeps width, height and frames as standard widgets. Its camera editor writes the existing motion/framing inputs, so saved workflows and connected inputs remain compatible.
- **Scene:** small first-frame previews of the actual layers. Drag to set camera travel; Play and the scrubber preview a six-second camera cycle. Center shows neutral framing without changing the render settings.
- **Depth order:** near-to-far layer diagram. Edit individual depths in the connected layer nodes or extraction review.
- **Camera:** Gentle, Reveal, Punchy and Still presets, motion curves, XY control, horizontal/vertical travel and push/pull. Presets change motion only.
- **Framing:** background distance, coverage, cutout cover/contain and render device. Contain preserves mismatched cutout aspect ratios. Background edges use clamped source pixels, not generated hidden scenery.
- **Relief:** Off, Background, or Background + Artwork, with strength, anchor depth, inversion and smoothing. Text layers from the poster stack remain flat. Zero strength preserves the previous renderer. The small GPU preview responds to strength/anchor/inversion; smoothing is applied on Run. Without WebGL, the preview remains flat.
Connect the stack to **FL Parallax Depth Sources**, then an image-batch depth estimator, then `depth_maps` on the compositor. Analysis sources use neutral gray behind cutout alpha and preserve stack order: background, then cutouts. A single depth map is accepted for background-only relief. This path supports still-image layers only. The workflow uses the installed Depth Anything V2 FP32 model at a 518-pixel analysis edge; the inference node is separate so camera and relief adjustments can reuse ComfyUI's cached maps.
Relief is a bounded inverse-depth warp inside each plate, not a mesh reconstruction. Color and premultiplied alpha share the same sampling grid. Plate draw order remains unchanged, and the Depth order output still visualizes flat layer ordering, not inferred per-pixel depth. Keep relief subtle near silhouettes, reflective objects, and narrow depth gaps. Depth on isolated cutouts can be ambiguous, and unseen surfaces are not reconstructed.
Before source previews are available, the editor displays labeled demo geometry. Landscape, typography and product-shape demos are preview-only. Actual previews use the same plane projection as the renderer, but reduced-resolution images, browser interpolation and frozen source frames mean they are not final-quality video. Upstream changes require Run to refresh the layers. Connected camera inputs are read-only in the editor and show their last rendered values.
Preview playback is limited to 20 drawn frames per second, starts paused, and stops scheduling animation when the node is offscreen or the browser tab is hidden. Source previews are at most 256 pixels on their longest edge and are saved under `output/Parallax/previews`.
The renderer computes a static scene's locked comparison once and shares that frame across the output batch. Animated input plates retain per-frame locked comparisons. Only the representative depth-preview frame is composited. No persistent GPU tensor cache is added.
## Different media
The Image Layer Planner follows the visible source medium. `artwork_type` guides grouping; it does not restyle the image. Auto mode adds text layers only when lettering is visible, retains actual environments, and can make one correction pass for an invalid or over-budget plan. Manual mode remains available for exact targets.
Use subtle travel for portraits, photography, reflective products and fine hair. Illustration and collage usually tolerate more separation; landscapes work best as coherent foreground/middle-ground/background groups. Large camera moves need reconstructed hidden regions or oversized source plates. Generated extraction can alter texture, lettering, shadows and transparency; inspect the RGBA layers before exporting.
+14
View File
@@ -0,0 +1,14 @@
# Scan FX layers
The **Layers** tab controls the digital back plates and cursor-window overlap. Existing workflows retain four cobalt plates, normal compositing, full reveal opacity, and no fades.
- **Stack count**: 0–8 plates. Spacing multiplies the original depth-relative offset; X/Y set direction, rotation fans each successive plate, and opacity affects plates and their outlines without fading the video.
- **Window order**: newest on top, a preferred effect on top, or seeded random ordering on Snare (Envelope 2 rising through 0.5). Random ranks hold between hits. They do not shuffle every frame.
- **Window blend**: normal, screen, or add. This is separate from Finish's glow blend mode. Per-effect opacity multiplies Reveal strength and the existing audio accent.
- **Fade in/out**: seconds for revealed content, not cursor graphics or borders. Fade-in starts when dragging begins; fade-out finishes on the gesture's last frame.
Drag Kick, Snare, or Hat onto spacing, X/Y spread, rotation, stack opacity, or any reveal opacity to assign an envelope. Count, palette, and ordering remain discrete settings. A useful starting mapping is Kick → stack spacing, minimum 0.6, maximum 2, smoothing 0.08 seconds.
The compact parameter panel uses two columns when space permits, `?` help, numeric ranges in control tooltips, and individual reset buttons. Mapping editors expand across both columns. Render-value meters appear only for assigned parameters.
**Live demo → Layers** approximates the stack and overlap on a synthetic scene. **All** includes the other effects. Actual envelopes, geometry, timing, and occlusion must still be checked in **Last render → Final**. These effects reuse existing images and maps; they do not run additional inference.
File diff suppressed because it is too large Load Diff
+29 -14
View File
@@ -3,6 +3,7 @@ import math
import re
from comfy_api.latest import io
from .prompt_references import reference_document, apply_reference_sections, reference_file_fingerprint
from .audio_files import (
audio_file_hash,
@@ -199,6 +200,8 @@ def _source_events(
if use_full_analysis:
data = source_analysis
if source in {"beat_grid", "downbeat", "raw_beat"}:
if not source_analysis.get("base_beat_times", source_analysis.get("beat_times", [])):
return []
data = apply_beat_offset(
apply_half_time(source_analysis, half_time),
fps,
@@ -286,13 +289,13 @@ def _parse_beat_payload(beat_positions):
raise ValueError(f"Beat positions is not valid JSON: {error.msg}.") from error
def _load_beat_data(beat_positions):
def _load_beat_data(beat_positions, allow_empty=False):
data = _parse_beat_payload(beat_positions)
if not isinstance(data, dict):
raise ValueError("Beat positions must be the JSON object from FL Audio BPM Analyzer.")
values = data.get("beat_times")
if not isinstance(values, list) or not values:
if not isinstance(values, list) or (not values and not allow_empty):
raise ValueError("Beat positions must contain a non-empty beat_times list.")
beat_times = []
@@ -305,7 +308,7 @@ def _load_beat_data(beat_positions):
duration = _number(data.get("audio_duration"), "audio_duration", 0)
if duration <= 0:
raise ValueError("Beat positions audio_duration must be greater than zero.")
if beat_times[-1] > duration + _EPS:
if beat_times and beat_times[-1] > duration + _EPS:
raise ValueError("Beat positions contains a beat after audio_duration.")
bpm = _number(data.get("bpm", 0.0), "bpm", 0)
@@ -726,6 +729,9 @@ def _frame_sections(sections, fps, total_frames):
}
if "render_group" in section:
frame_section["render_group"] = section["render_group"]
for key in ("section_id", "references"):
if key in section:
frame_section[key] = section[key]
frame_sections.append(frame_section)
return frame_sections
@@ -903,6 +909,8 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
"the three fixed envelope slots automatically."
),
),
io.String.Input("reference_schedule", default="", optional=True,
tooltip="Sequencer-owned section references and asset manifest."),
],
outputs=[
FLPromptSchedule.Output(
@@ -960,6 +968,7 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
render_groups="",
analysis_cache_key="",
envelope_layers="",
reference_schedule="",
):
internal_analysis = None
cropped_audio = None
@@ -980,15 +989,16 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
)
if beat_positions:
beat_payload = _parse_beat_payload(beat_positions)
_load_beat_data(beat_payload)
beat_payload = apply_beat_offset(
beat_payload,
fps,
beat_offset_ms,
beat_grid_density,
)
_load_beat_data(beat_payload, allow_empty=time_unit != "beats")
if beat_payload.get("base_beat_times", beat_payload.get("beat_times", [])):
beat_payload = apply_beat_offset(
beat_payload,
fps,
beat_offset_ms,
beat_grid_density,
)
beat_positions = json.dumps(beat_payload, separators=(",", ":"))
beat_data = _load_beat_data(beat_payload)
beat_data = _load_beat_data(beat_payload, allow_empty=time_unit != "beats")
if internal_analysis is not None:
difference = abs(beat_data["audio_duration"] - internal_analysis["audio_duration"])
if difference > max(_EPS, 1.0 / fps):
@@ -998,7 +1008,7 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
)
elif internal_analysis is not None:
beat_positions = json.dumps(internal_analysis, separators=(",", ":"))
beat_data = _load_beat_data(beat_positions)
beat_data = _load_beat_data(beat_positions, allow_empty=time_unit != "beats")
else:
raise ValueError("Choose an audio file or connect beat_positions.")
@@ -1028,6 +1038,8 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
),
render_groups,
)
references = reference_document(reference_schedule, len(parsed_sections))
apply_reference_sections(parsed_sections, references)
sections = _resolve_schedule(
parsed_sections,
beat_times,
@@ -1062,6 +1074,7 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
"source_unit": time_unit,
"fps": fps,
"sections": sections,
"reference_assets": references["assets"],
}
ui_payload = {
"bpm": beat_data["bpm"],
@@ -1153,15 +1166,17 @@ class FL_Audio_Beat_Prompt_Schedule(io.ComfyNode):
audio_file="",
analysis_cache_key="",
beat_positions=None,
reference_schedule="",
**kwargs,
):
references = reference_file_fingerprint(reference_schedule)
if not audio_file and analysis_cache_key:
audio_file = cached_analysis_audio_file(analysis_cache_key)
if not audio_file:
return None
return (None, references)
analysis_version = (
f"audio-timeline-{ANALYSIS_VERSION}"
if beat_positions
else f"{DETECTOR_VERSION}:timeline-{ANALYSIS_VERSION}"
)
return f"{analysis_version}:{audio_file_hash(resolve_audio_path(audio_file))}"
return (f"{analysis_version}:{audio_file_hash(resolve_audio_path(audio_file))}", references)
+28 -4
View File
@@ -28,21 +28,25 @@ class FL_Audio_Drum_Detector:
"min": 0.0,
"max": 1.0,
"step": 0.05,
"description": "Kick detection sensitivity (lower = more sensitive)"
"description": "Kick detection sensitivity (higher = more sensitive)"
}),
"snare_sensitivity": ("FLOAT", {
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.05,
"description": "Snare detection sensitivity (lower = more sensitive)"
"description": "Snare detection sensitivity (higher = more sensitive)"
}),
"hihat_sensitivity": ("FLOAT", {
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.05,
"description": "Hi-hat detection sensitivity (lower = more sensitive)"
"description": "Hi-hat detection sensitivity (higher = more sensitive)"
}),
"detection_mode": (["classified_onsets", "independent_bands"], {
"default": "classified_onsets",
"description": "Independent bands detect low/mid/high transients separately for dense full mixes; these are frequency accents, not isolated instrument labels."
}),
}
}
@@ -52,7 +56,8 @@ class FL_Audio_Drum_Detector:
audio: Dict[str, Any],
kick_sensitivity: float = 0.5,
snare_sensitivity: float = 0.5,
hihat_sensitivity: float = 0.5
hihat_sensitivity: float = 0.5,
detection_mode: str = "classified_onsets"
) -> Tuple[str]:
"""
Detect drum elements from audio
@@ -96,6 +101,25 @@ class FL_Audio_Drum_Detector:
print(f"[FL Audio Drum Detector] DEBUG: Waveform shape = {waveform_np.shape}")
print(f"[FL Audio Drum Detector] DEBUG: Sample rate = {sample_rate}")
if detection_mode == "independent_bands":
spectrum = np.abs(librosa.stft(waveform_np))
frequencies = librosa.fft_frequencies(sr=sample_rate)
data = {"sample_rate": int(sample_rate), "duration": float(len(waveform_np) / sample_rate),
"detection_mode": "independent_bands"}
for name, low, high, sensitivity in (("kick", 30, 180, kick_sensitivity),
("snare", 180, 5000, snare_sensitivity), ("hihat", 6000, 16000, hihat_sensitivity)):
band = spectrum[(frequencies >= low) & (frequencies < high)]
flux = np.zeros(spectrum.shape[1], dtype=np.float32)
if band.size:
flux[1:] = np.maximum(np.diff(band, axis=1), 0).mean(axis=0)
times = librosa.onset.onset_detect(onset_envelope=flux, sr=sample_rate,
hop_length=512, units="time", delta=0.05 + (1 - sensitivity) * 0.2,
wait=max(1, round(0.09 * sample_rate / 512)))
data[f"{name}_times"] = times.tolist()
data[f"total_{'hihats' if name == 'hihat' else name + 's'}"] = len(times)
print(f"[FL Audio Drum Detector] Band onsets: {data['total_kicks']} low, {data['total_snares']} mid, {data['total_hihats']} high")
return (json.dumps(data, indent=2),)
# Detect onsets
print(f"[FL Audio Drum Detector] Detecting onsets...")
onset_env = librosa.onset.onset_strength(y=waveform_np, sr=sample_rate)
+19 -20
View File
@@ -150,8 +150,7 @@ class FL_Audio_Separation:
import traceback
traceback.print_exc()
print(f"{'='*60}\n")
# Return original audio for all outputs on error
return (audio, audio, audio, audio)
raise RuntimeError(f"FL Audio Separation failed: {e}") from e
def _ensure_stereo(self, waveform: torch.Tensor) -> torch.Tensor:
"""Ensure waveform is stereo"""
@@ -196,32 +195,32 @@ class FL_Audio_Separation:
batch, channels, length = mix.shape
chunk_len = int(sample_rate * segment * (1 + overlap))
chunk_len = round(sample_rate * segment)
overlap_frames = round(overlap * sample_rate)
if not 0 <= overlap_frames < chunk_len:
raise ValueError("Stem overlap must be shorter than the chunk length.")
start = 0
end = chunk_len
overlap_frames = overlap * sample_rate
fade = Fade(
fade_in_len=0,
fade_out_len=int(overlap_frames),
fade_shape=chunk_fade_shape
)
final = torch.zeros(batch, len(model.sources), channels, length, device=device)
final = torch.zeros(batch, len(model.sources), channels, length, device=device, dtype=mix.dtype)
weights = torch.zeros(length, device=device, dtype=mix.dtype)
while start < length - overlap_frames:
while start < length:
end = min(length, start + chunk_len)
chunk = mix[:, :, start:end]
with torch.no_grad():
out = model.forward(chunk)
out = fade(out)
final[:, :, :, start:end] += out
out = model(chunk)
fade.fade_in_len = min(overlap_frames, end - start) if start else 0
fade.fade_out_len = min(overlap_frames, end - start) if end < length else 0
weight = fade(torch.ones(end - start, device=device, dtype=mix.dtype))
final[..., start:end].add_(out * weight)
weights[start:end].add_(weight)
if end == length:
break
start += chunk_len - overlap_frames
if start == 0:
fade.fade_in_len = int(overlap_frames)
start += int(chunk_len - overlap_frames)
else:
start += chunk_len
end += chunk_len
if end >= length:
fade.fade_out_len = 0
return final
return final / weights.clamp_min(1e-8)
@@ -0,0 +1,45 @@
import numpy as np
import torch
from PIL import Image, ImageOps
from comfy_api.latest import InputImpl
from comfy_extras.nodes_audio import load as load_audio
from .prompt_references import reference_path
class FL_Prompt_Reference_Library:
@classmethod
def INPUT_TYPES(cls):
return {"required": {"prompt_schedule": ("FL_PROMPT_SCHEDULE",)}}
RETURN_TYPES = ("FL_PROMPT_REFERENCES",)
RETURN_NAMES = ("reference_library",)
FUNCTION = "load"
CATEGORY = "🏵️Fill Nodes/Audio"
def load(self, prompt_schedule):
assets = prompt_schedule.get("reference_assets", {})
selected = dict.fromkeys(asset_id for section in prompt_schedule["sections"]
for asset_id in section.get("references", {}).get("asset_ids", []))
resolved = {}
for asset_id in selected:
asset = assets[asset_id]
path = reference_path(asset)
kind = asset["kind"]
if kind == "image":
with Image.open(path) as image:
if image.width * image.height > 64_000_000:
raise ValueError("Reference image exceeds the 64-megapixel limit.")
rgb = ImageOps.exif_transpose(image).convert("RGB")
value = torch.from_numpy(np.array(rgb).astype(np.float32) / 255.0).unsqueeze(0)
resolved[asset_id] = {"kind": kind, "value": value}
elif kind == "audio":
waveform, rate = load_audio(str(path))
resolved[asset_id] = {"kind": kind, "value": {"waveform": waveform.unsqueeze(0), "sample_rate": rate}}
elif kind == "video":
components = InputImpl.VideoFromFile(str(path)).get_components()
resolved[asset_id] = {"kind": kind, "value": components.images, "audio": components.audio, "fps": float(components.frame_rate)}
else:
raise ValueError(f"Unsupported reference kind: {kind}")
return ({"version": 1, "assets": resolved},)
+82
View File
@@ -0,0 +1,82 @@
import json
from pathlib import Path
import folder_paths
def reference_document(value, section_count):
if not value:
return {"version": 1, "assets": {}, "sections": []}
document = json.loads(value) if isinstance(value, str) else value
if not isinstance(document, dict) or document.get("version") != 1:
raise ValueError("Unsupported prompt reference document version.")
assets = document.get("assets", {})
sections = document.get("sections", [])
if not isinstance(assets, dict) or len(assets) > 2048:
raise ValueError("Reference library must contain at most 2048 assets.")
if not isinstance(sections, list) or len(sections) != section_count:
raise ValueError("Reference sections no longer match the timeline. Reopen the sequencer.")
seen = set()
for section in sections:
if not isinstance(section, dict):
raise ValueError("Invalid reference section.")
section_id = section.get("id")
if not isinstance(section_id, str) or not section_id or section_id in seen:
raise ValueError("Reference sections require unique, stable IDs.")
seen.add(section_id)
mode = section.get("mode", "defaults")
ids = section.get("asset_ids", [])
if mode not in {"defaults", "custom", "none"} or not isinstance(ids, list):
raise ValueError("Invalid section reference selection.")
if any(not isinstance(asset_id, str) for asset_id in ids) or len(ids) != len(set(ids)) or any(asset_id not in assets for asset_id in ids):
raise ValueError("Section references contain missing or duplicate assets.")
if mode != "custom" and ids:
raise ValueError("Only custom reference sections may select assets.")
for asset_id, asset in assets.items():
if not isinstance(asset_id, str) or not isinstance(asset, dict):
raise ValueError("Invalid reference asset.")
if asset.get("kind") not in {"image", "video", "audio"}:
raise ValueError(f"Reference {asset_id} has an unsupported media type.")
reference_path(asset, must_exist=False)
return document
def reference_file_fingerprint(value):
if not value:
return ()
document = json.loads(value) if isinstance(value, str) else value
document = reference_document(document, len(document.get("sections", [])))
selected = dict.fromkeys(asset_id for section in document["sections"] for asset_id in section.get("asset_ids", []))
files = []
for asset_id in selected:
path = reference_path(document["assets"][asset_id])
stat = path.stat()
files.append((str(path), stat.st_mtime_ns, stat.st_size))
return tuple(files)
def reference_path(asset, must_exist=True):
roots = {"input": folder_paths.get_input_directory(), "output": folder_paths.get_output_directory()}
storage = asset.get("type", "input")
if storage not in roots:
raise ValueError("References must be stored in ComfyUI input or output.")
filename = asset.get("filename", "")
subfolder = asset.get("subfolder", "")
if not isinstance(filename, str) or not filename or Path(filename).name != filename or any(character in filename for character in "/\\:"):
raise ValueError("Invalid reference filename.")
if not isinstance(subfolder, str):
raise ValueError("Invalid reference subfolder.")
root = Path(roots[storage]).resolve()
path = (root / subfolder / filename).resolve()
if not path.is_relative_to(root) or path == root:
raise ValueError("Reference path must stay inside its ComfyUI media directory.")
if must_exist and not path.is_file():
raise ValueError(f"Missing reference file: {filename}. Copy the workflow's reference media into ComfyUI.")
return path
def apply_reference_sections(sections, document):
for section, reference in zip(sections, document["sections"]):
section["section_id"] = reference["id"]
section["references"] = {"mode": reference.get("mode", "defaults"), "asset_ids": list(reference.get("asset_ids", []))}
return sections
+62
View File
@@ -0,0 +1,62 @@
STORYBOARD_SCHEMA = {
"type": "array",
"description": "Image generation requests, queued automatically with ComfyUI credits when the Writer finishes. Empty unless the user requests images. Prefer 2x2 for sections up to six seconds; 3x3 for longer sections. Describe chronological panel beats. Across all requests repeat the same precise cast identity, clothing, palette, drawing technique and world design, honoring selected moodboard roles. Change only action, framing and explicitly requested story changes; do not redesign characters between sections.",
"items": {"type": "object", "properties": {
"index": {"type": "integer", "minimum": 0},
"grid": {"type": "integer", "enum": [2, 3]},
"prompt": {"type": "string", "minLength": 1},
}, "required": ["index", "grid", "prompt"], "additionalProperties": False},
}
REFERENCE_ASSIGNMENT_SCHEMA = {
"type": "array", "maxItems": 8,
"description": "Apply these reference assignments directly to scoped sections. Use only asset IDs supplied in the reference library context.",
"items": {"type": "object", "properties": {
"index": {"type": "integer", "minimum": 0},
"mode": {"type": "string", "enum": ["defaults", "custom", "none"]},
"asset_ids": {"type": "array", "items": {"type": "string"}},
}, "required": ["index", "mode", "asset_ids"], "additionalProperties": False},
}
def normalize_asset_ids(values):
if not isinstance(values, list) or len(values) > 2048 or any(not isinstance(value, str) or not 1 <= len(value) <= 128 for value in values):
raise ValueError("Invalid reference library IDs.")
return list(dict.fromkeys(values))
def normalize_reference_assignments(values, allowed_indices, asset_ids):
if not isinstance(values, list) or len(values) > 8:
raise ValueError("Request at most eight reference assignments.")
result = []
seen = set()
for value in values:
if not isinstance(value, dict):
raise ValueError("Invalid reference assignment.")
index, mode = value.get("index"), value.get("mode")
ids = normalize_asset_ids(value.get("asset_ids", []))
if type(index) is not int or index not in allowed_indices or index in seen:
raise ValueError("Reference assignment targets an unavailable or duplicate section.")
if mode not in {"defaults", "custom", "none"} or (mode != "custom" and ids) or any(asset_id not in asset_ids for asset_id in ids):
raise ValueError("Reference assignment uses unavailable assets or an invalid mode.")
seen.add(index)
result.append({"index": index, "mode": mode, "asset_ids": ids})
return result
def normalize_storyboard_actions(values, allowed_indices):
if not isinstance(values, list):
raise ValueError("Storyboard requests must be a list.")
result = []
seen = set()
for value in values:
if not isinstance(value, dict):
raise ValueError("Invalid storyboard request.")
index, grid, prompt = value.get("index"), value.get("grid"), value.get("prompt")
if type(index) is not int or index not in allowed_indices or index in seen:
raise ValueError("Storyboard request targets an unavailable or duplicate section.")
if type(grid) is not int or grid not in (2, 3) or not isinstance(prompt, str) or not 1 <= len(prompt.strip()) <= 64000:
raise ValueError("Storyboard requests need a 2x2 or 3x3 grid and a bounded prompt.")
seen.add(index)
result.append({"index": index, "grid": grid, "prompt": prompt.strip()})
return result
+204
View File
@@ -0,0 +1,204 @@
import json
import sqlite3
import uuid
from PIL import Image
from .prompt_references import reference_path
from .prompt_writer_config import DATA_DIR
MODEL = "Nano Banana 2 (Gemini 3.1 Flash Image)"
def validate_storyboard(value):
if not isinstance(value, dict):
raise ValueError("Storyboard must be an object.")
result = {}
for key in ("scheduler_id", "section_id", "revision", "prompt"):
text = value.get(key)
if not isinstance(text, str) or not text.strip() or len(text) > (64000 if key == "prompt" else 128):
raise ValueError(f"Invalid storyboard {key}.")
result[key] = text
request_key = value.get("request_key")
continuity = value.get("continuity", "")
if not isinstance(continuity, str) or len(continuity) > 32000:
raise ValueError("Storyboard continuity brief must be at most 32000 characters.")
result["continuity"] = continuity
if request_key is not None:
if not isinstance(request_key, str) or not 1 <= len(request_key) <= 256:
raise ValueError("Invalid storyboard request key.")
result["request_key"] = request_key
grid = value.get("grid", 2)
if type(grid) is not int or grid not in (2, 3):
raise ValueError("Choose a 2×2 or 3×3 storyboard.")
resolution = value.get("resolution", "2K")
aspect = value.get("aspect_ratio", "16:9")
if resolution not in ("1K", "2K", "4K") or aspect not in ("16:9", "9:16", "1:1", "4:3", "3:4"):
raise ValueError("Unsupported storyboard resolution or aspect ratio.")
moodboards = value.get("moodboards", [])
if not isinstance(moodboards, list) or len(moodboards) > 4:
raise ValueError("Select at most four moodboards.")
selected = []
for image in moodboards:
if not isinstance(image, dict) or not isinstance(image.get("role", ""), str) or len(image.get("role", "")) > 500:
raise ValueError("Invalid moodboard role.")
reference_path(image)
if image.get("type", "input") != "input":
raise ValueError("Moodboards must be uploaded to ComfyUI input.")
selected.append({"filename": image["filename"], "subfolder": image.get("subfolder", ""), "type": "input", "role": image.get("role", "")})
return {**result, "grid": grid, "resolution": resolution, "aspect_ratio": aspect, "moodboards": selected}
def storyboard_graph(job_id, spec):
inputs = {
"prompt": f"Create one {spec['grid']}x{spec['grid']} storyboard contact sheet. Equal rectangular cells, no gutters, labels, captions, text or timestamps. Read left to right, top to bottom: each cell progresses chronologically through this section. Keep character identity and style consistent.\n{spec['prompt']}",
"model": MODEL,
"model.aspect_ratio": spec["aspect_ratio"],
"model.resolution": spec["resolution"],
"model.thinking_level": "MINIMAL",
"seed": int(uuid.UUID(job_id)) % (2**32),
"response_modalities": "IMAGE",
}
graph = {}
if spec.get("continuity"):
inputs["prompt"] += "\nShared production continuity (context only; depict only the requested section):\n" + spec["continuity"]
inputs["prompt"] += "\nPreserve the same cast identity, face, hair, costume, palette, drawing technique and world design across sections unless the section explicitly requests a change. Treat supplied character/style reference images as visual anchors, not optional inspiration."
for index, image in enumerate(spec["moodboards"], 1):
node_id = f"moodboard_{index}"
name = "/".join(part for part in (image["subfolder"], image["filename"]) if part)
graph[node_id] = {"class_type": "LoadImage", "inputs": {"image": name + " [input]"}}
inputs[f"model.images.image_{index}"] = [node_id, 0]
if image.get("role"):
inputs["prompt"] += f"\nReference image {index} role: {image['role']}"
graph["storyboard"] = {"class_type": "GeminiNanoBanana2V2", "inputs": inputs}
graph["save_storyboard"] = {"class_type": "SaveImage", "inputs": {
"images": ["storyboard", 0], "filename_prefix": f"fl-storyboards/{job_id}/sheet"}}
return graph
def storyboard_batch_graph(jobs):
graph = {}
for job in jobs:
branch = storyboard_graph(job["id"], job["spec"])
prefix = job["id"] + ":"
for node_id, node in branch.items():
graph[prefix + node_id] = {**node, "inputs": {key: [prefix + value[0], value[1]]
if isinstance(value, list) and len(value) == 2 and value[0] in branch else value
for key,value in node["inputs"].items()}}
return graph
class StoryboardStore:
def __init__(self, path=None):
self.path = path or DATA_DIR / "storyboards.db"
def connect(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(self.path)
connection.execute("CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, scheduler TEXT, state TEXT, spec TEXT, result TEXT)")
return connection
def create(self, value):
spec = validate_storyboard(value)
job_id = str(uuid.uuid5(uuid.NAMESPACE_URL, json.dumps([spec["scheduler_id"], spec["request_key"]]))) if "request_key" in spec else str(uuid.uuid4())
connection = self.connect()
try:
with connection:
connection.execute("INSERT OR IGNORE INTO jobs VALUES (?, ?, 'proposed', ?, '{}')", (job_id, spec["scheduler_id"], json.dumps(spec)))
finally:
connection.close()
return self.get(job_id)
def get(self, job_id):
connection = self.connect()
try:
row = connection.execute("SELECT id, state, spec, result FROM jobs WHERE id=?", (job_id,)).fetchone()
finally:
connection.close()
if row is None:
raise ValueError("Storyboard job not found.")
return {"id": row[0], "state": row[1], "spec": json.loads(row[2]), "result": json.loads(row[3])}
def list(self, scheduler):
connection = self.connect()
try:
ids = connection.execute("SELECT id FROM jobs WHERE scheduler=? ORDER BY rowid DESC LIMIT 100", (scheduler,)).fetchall()
finally:
connection.close()
return [self.get(row[0]) for row in ids]
def claim(self, job_id):
job = self.get(job_id)
validate_storyboard(job["spec"])
connection = self.connect()
try:
with connection:
changed = connection.execute("UPDATE jobs SET state='submitted' WHERE id=? AND state='proposed'", (job_id,)).rowcount
finally:
connection.close()
if not changed:
raise ValueError("This job was already submitted or cancelled. Refresh its status; do not resubmit.")
return storyboard_graph(job_id, job["spec"])
def claim_batch(self, job_ids):
connection = self.connect()
try:
with connection:
for job_id in job_ids:
changed = connection.execute("UPDATE jobs SET state='submitted' WHERE id=? AND state='proposed'", (job_id,)).rowcount
if not changed:
raise ValueError("A storyboard was already submitted or cancelled. Refresh before trying again.")
finally:
connection.close()
def update(self, job_id, state, result):
connection = self.connect()
try:
with connection:
connection.execute("UPDATE jobs SET state=?, result=? WHERE id=?", (state, json.dumps(result), job_id))
finally:
connection.close()
return self.get(job_id)
def extract_panels(job, source, bounds=None):
source_path = reference_path(source)
expected = f"fl-storyboards/{job['id']}"
if source.get("type") != "output" or source.get("subfolder", "").replace("\\", "/") != expected:
raise ValueError("The image is not an output of this storyboard job.")
grid = job["spec"]["grid"]
with Image.open(source_path) as image:
image = image.convert("RGB")
width, height = image.size
if bounds is None:
bounds = [0, 0, width, height]
if not isinstance(bounds, list) or len(bounds) != 4 or any(type(value) is not int for value in bounds):
raise ValueError("Crop bounds must be four integer pixel coordinates.")
left, top, right, bottom = bounds
if not (0 <= left < right <= width and 0 <= top < bottom <= height) or min(right-left, bottom-top) < grid:
raise ValueError("Crop bounds must be inside the contact sheet.")
version = uuid.uuid4().hex
assets = {}
for index in range(grid * grid):
row, column = divmod(index, grid)
box = (left + (right-left)*column//grid, top + (bottom-top)*row//grid,
left + (right-left)*(column+1)//grid, top + (bottom-top)*(row+1)//grid)
panel = image.crop(box)
filename = f"panel-{version}-{index+1}.png"
panel.save(source_path.parent / filename)
assets[f"{version}-{index+1}"] = {
"kind": "image", "filename": filename, "subfolder": expected, "type": "output",
"label": f"Storyboard panel {index+1}", "width": panel.width, "height": panel.height,
"storyboard_id": job["id"], "version": version, "panel": index+1,
"source": source,
"generation": {"model": MODEL, "grid": grid, "resolution": job["spec"]["resolution"],
"aspect_ratio": job["spec"]["aspect_ratio"], "section_id": job["spec"]["section_id"], "revision": job["spec"]["revision"]},
"section_moment": index / max(1, grid*grid-1),
}
return {"source": source, "assets": assets, "bounds": bounds,
"versions": [*job["result"].get("versions", []), {"version": version, "assets": assets, "bounds": bounds}],
"warning": "Panels below 512 pixels may lose detail; consider a larger contact sheet." if min(right-left, bottom-top)//grid < 512 else ""}
storyboard_store = StoryboardStore()
+28
View File
@@ -5,6 +5,7 @@ from pathlib import Path
from urllib.parse import urlsplit, urlunsplit
import aiohttp
from .prompt_storyboard_actions import STORYBOARD_SCHEMA, REFERENCE_ASSIGNMENT_SCHEMA, normalize_storyboard_actions, normalize_reference_assignments, normalize_asset_ids
MAX_BOXES = 256
@@ -50,12 +51,24 @@ GUIDE_INSTRUCTIONS = {
def prompt_writing_instructions(guide_mode):
return (
"When asked to generate storyboard references, request chronological contact sheets using the storyboard schema; "
"the host queues paid Nano Banana 2 images automatically after your response and attaches the panels to each section. Do not claim images already exist. "
"Prefer 2x2 for up to six seconds and 3x3 for longer sections. "
+
GUIDE_INSTRUCTIONS[guide_mode]
+ "\n\nComplete packaged prompt-writing guide:\n\n"
+ PROMPT_WRITING_GUIDE
)
TOOLS = [
{"type": "function", "function": {
"name": "set_reference_assignments", "description": "Assign scoped section references using available library IDs.",
"parameters": {"type": "object", "properties": {"reference_assignments": REFERENCE_ASSIGNMENT_SCHEMA}, "required": ["reference_assignments"], "additionalProperties": False},
}},
{"type": "function", "function": {
"name": "generate_storyboards", "description": "Request section storyboard images. The host queues generation using ComfyUI credits after this turn, without another approval dialog.",
"parameters": {"type": "object", "properties": {"storyboards": STORYBOARD_SCHEMA}, "required": ["storyboards"], "additionalProperties": False},
}},
{
"type": "function",
"function": {
@@ -618,6 +631,7 @@ def _normalize_request(value):
"guide_mode": guide_mode,
"messages": _normalize_messages(value.get("messages")),
"boxes": _normalize_boxes(value.get("boxes")),
"reference_assets": normalize_asset_ids(value.get("reference_assets", [])),
}
@@ -998,6 +1012,8 @@ async def run_prompt_writer(
timeout = aiohttp.ClientTimeout(total=180, connect=10)
session = aiohttp.ClientSession(timeout=timeout)
tool_calls_used = 0
storyboards = []
reference_assignments = []
final_text = ""
target_indices = None
streamed_indices = set()
@@ -1053,6 +1069,8 @@ async def run_prompt_writer(
"label": (
"Reading prompt boxes"
if name == "get_prompt_boxes"
else "Preparing references"
if name in {"generate_storyboards", "set_reference_assignments"}
else "Planning prompt edits"
if name == "plan_prompt_boxes"
else "Updating prompt boxes"
@@ -1075,6 +1093,12 @@ async def run_prompt_writer(
result = {"planned": target_indices, "count": len(target_indices)}
if on_prompt_progress:
await on_prompt_progress({"type": "plan", "target_indices": target_indices})
elif name == "generate_storyboards":
storyboards = normalize_storyboard_actions(storyboards + arguments.get("storyboards", []), boxes_by_index)
result = {"count": len(storyboards), "status": "ready for automatic generation after this turn"}
elif name == "set_reference_assignments":
reference_assignments = normalize_reference_assignments(reference_assignments + arguments.get("reference_assignments", []), boxes_by_index, request["reference_assets"])
result = {"count": len(reference_assignments), "status": "ready for automatic assignment after this turn"}
elif name == "set_prompt_boxes":
normalized_updates = _normalize_tool_updates(arguments, boxes_by_index)
if target_indices is None:
@@ -1104,6 +1128,8 @@ async def run_prompt_writer(
"label": (
f"Read {count} prompt box{'es' if count != 1 else ''}"
if name == "get_prompt_boxes"
else f"Prepared {count} reference action(s)"
if name in {"generate_storyboards", "set_reference_assignments"}
else f"Planned {count} prompt edit{'s' if count != 1 else ''}"
if name == "plan_prompt_boxes"
else f"Updated {count} prompt box{'es' if count != 1 else ''}"
@@ -1145,4 +1171,6 @@ async def run_prompt_writer(
"updates": updates,
"target_indices": target_indices or [],
"tool_calls": tool_calls_used,
"storyboards": storyboards,
"reference_assignments": reference_assignments,
}
+29 -3
View File
@@ -40,6 +40,7 @@ from .prompt_writer_images import (
normalize_prompt_writer_attachments,
)
from .prompt_writer_store import PromptWriterStore, prompt_writer_store
from .prompt_storyboard_actions import STORYBOARD_SCHEMA, REFERENCE_ASSIGNMENT_SCHEMA, normalize_storyboard_actions, normalize_reference_assignments, normalize_asset_ids
logger = logging.getLogger("fl_fill_nodes.prompt_writer")
@@ -51,6 +52,8 @@ CLAUDE_MAX_MESSAGE_BYTES = 8 * 1024 * 1024
STRUCTURED_RESULT_SCHEMA = {
"type": "object",
"properties": {
"storyboards": STORYBOARD_SCHEMA,
"reference_assignments": REFERENCE_ASSIGNMENT_SCHEMA,
"target_indices": {
"type": "array",
"maxItems": MAX_BOXES,
@@ -77,7 +80,7 @@ STRUCTURED_RESULT_SCHEMA = {
"description": "A concise conversational response explaining the result.",
},
},
"required": ["target_indices", "updates", "assistant"],
"required": ["target_indices", "updates", "assistant", "storyboards", "reference_assignments"],
"additionalProperties": False,
}
@@ -118,6 +121,7 @@ def _normalize_document(value):
allow_empty=True,
),
"boxes": _normalize_boxes(value.get("boxes")),
"reference_assets": normalize_asset_ids(value.get("reference_assets", [])),
}
@@ -205,6 +209,11 @@ def _structured_prompt(document, messages, vision_images=None):
def _structured_system_prompt(guide_mode):
return (
"Request storyboard reference generation using the storyboards array when asked. "
"The host queues paid contact sheets automatically after your response and attaches their panels to the timeline. Do not claim images have finished yet. "
"Use chronological panel beats, consistent identity and no text in the image. "
"You can also assign reference_assignments from supplied library IDs directly to scoped sections. "
"Prefer a 2x2 grid for up to six seconds and 3x3 for longer sections. Use only supplied box indices. "
"You are Beat Writer, a prompt-writing agent embedded in an audio beat prompt scheduler. "
"You may work only on the prompt boxes supplied with the current request. Never change or "
"invent timing, frame ranges, fades, render groups, audio settings, nodes, files, or workflow "
@@ -271,6 +280,19 @@ def _parse_structured_result(value):
return assistant.strip(), updates, target_indices
def _structured_actions(value, document):
if isinstance(value, str):
text = value.strip()
if text.startswith("```"):
text = text.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
value = json.loads(text)
indices = {box["index"] for box in document["boxes"]}
return {
"storyboards": normalize_storyboard_actions(value.get("storyboards", []), indices),
"reference_assignments": normalize_reference_assignments(value.get("reference_assignments", []), indices, document["reference_assets"]),
}
def _native_claude_cli(path):
if not path or Path(path).suffix.lower() in {".bat", ".cmd"}:
return None
@@ -780,9 +802,12 @@ class PromptWriterRuntime:
"song_context": run.document["song_context"],
"lyrics_context": run.document["lyrics_context"],
"boxes": run.document["boxes"],
"reference_assets": run.document["reference_assets"],
}, on_text_delta=on_text_delta, on_tool_event=on_tool_event, on_prompt_progress=on_prompt_progress,
vision_images=vision_images)
return result["assistant"], result["updates"], {
"storyboards": result.get("storyboards", []),
"reference_assignments": result.get("reference_assignments", []),
"toolCalls": result["tool_calls"],
"_target_indices": result["target_indices"],
"_tools_published": True,
@@ -811,6 +836,7 @@ class PromptWriterRuntime:
text = "".join(block.text for block in response.content if getattr(block, "type", None) == "text")
assistant, updates, target_indices = _parse_structured_result(text)
return assistant, updates, {
**_structured_actions(text, run.document),
"usage": getattr(response, "usage", None).model_dump() if response.usage else {},
"_target_indices": target_indices,
}
@@ -874,7 +900,7 @@ class PromptWriterRuntime:
if result is None:
raise PromptWriterProviderError("Claude subscription returned no result.")
assistant, updates, target_indices = _parse_structured_result(result)
return assistant, updates, {"_target_indices": target_indices}
return assistant, updates, {"_target_indices": target_indices, **_structured_actions(result, run.document)}
async def _run_codex(self, run, messages):
from openai_codex import ApprovalMode, AsyncCodex, CodexConfig, Sandbox
@@ -974,7 +1000,7 @@ class PromptWriterRuntime:
detail = completed_turn.error.message if completed_turn.error else "Codex turn failed."
raise PromptWriterProviderError(detail)
assistant, updates, target_indices = _parse_structured_result(completed_text)
return assistant, updates, {"usage": usage, "_target_indices": target_indices}
return assistant, updates, {"usage": usage, "_target_indices": target_indices, **_structured_actions(completed_text, run.document)}
prompt_writer_runtime = PromptWriterRuntime()
+205
View File
@@ -0,0 +1,205 @@
import math
import comfy.model_base
import comfy.samplers
import comfy.utils
from comfy.text_encoders.krea2 import KREA2_TEMPLATE, Krea2Tokenizer
from comfy_api.latest import io
Reference = io.Custom("FL_KREA_REFERENCE")
ROLES = {
"style": "Borrow the reference's medium, linework, shading, and texture. Use the scene below for the subject and composition, not the reference's subjects or panel layout.",
"palette": "Borrow the reference's colors and color relationships. Use the scene below for the subjects, composition, and rendering style.",
"subject": "Use the appearance of the reference's main subject in the scene below. Follow the scene for the setting and composition.",
"composition": "Borrow the reference's broad arrangement, framing, and spatial relationships. Use the scene below for the subjects and appearance.",
"custom": "",
}
RESOLUTIONS = (256, 512, 1024, 1280)
def encode_reference(clip, prompt, reference):
image = reference["image"]
height, width = image.shape[1:3]
scale = min(1.0, reference["resolution"] / max(height, width))
# Qwen3-VL merges 2x2 patches of 16 pixels; preserve at least one merged patch per axis.
width, height = max(32, round(width * scale)), max(32, round(height * scale))
image = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "area", "disabled").movedim(1, -1)
instruction = ROLES[reference["role"]]
text = f"<|vision_start|><|image_pad|><|vision_end|>\n{instruction}\nCreate one coherent image of this scene:\n{prompt}"
tokens = clip.tokenize(text, images=[image], llama_template=KREA2_TEMPLATE)
conditioning = clip.encode_from_tokens_scheduled(tokens)
if reference["reference_mode"] == "context":
pairs = tokens["qwen3vl_4b"][0]
image_end = next(i for i, pair in enumerate(pairs) if isinstance(pair[0], int) and pair[0] == 151653)
suffix_length = len(pairs) - image_end - 1
# Keep text states that attended to the image, without passing the image's spatial tokens to the DiT.
result = []
for tensor, metadata in conditioning:
metadata = metadata.copy()
if "attention_mask" in metadata:
metadata["attention_mask"] = metadata["attention_mask"][:, -suffix_length:].clone()
result.append([tensor[:, -suffix_length:].clone(), metadata])
conditioning = result
return conditioning
def sigma_envelope(sigma, bounds):
start, fade_in, fade_out, end = bounds
if sigma > start or sigma < end:
return 0.0
gain = 1.0
if start > fade_in and sigma > fade_in:
gain = (start - sigma) / (start - fade_in)
if fade_out > end and sigma < fade_out:
gain = min(gain, (sigma - end) / (fade_out - end))
return gain
class KreaReferenceGuider(comfy.samplers.CFGGuider):
def __init__(self, model, baseline, branches, influence):
super().__init__(model)
self.influence = influence
sampling = model.get_model_object("model_sampling")
self.references = []
conditions = {"positive": baseline}
for i, (reference, conditioning) in enumerate(branches):
name = f"reference_{i}"
start, end = reference["start"], reference["end"]
fade = (end - start) * reference["fade"]
bounds = tuple(float(sampling.percent_to_sigma(t)) for t in (start, start + fade, end - fade, end))
self.references.append((name, reference["weight"], bounds))
conditions[name] = conditioning
self.inner_set_conds(conditions)
def predict_noise(self, x, timestep, model_options={}, seed=None):
if not self.references or self.influence == 0:
return super().predict_noise(x, timestep, model_options, seed)
sigma = float(timestep[0])
weighted = [(name, self.influence * weight * sigma_envelope(sigma, bounds)) for name, weight, bounds in self.references]
baseline_weight = 1.0 - math.fsum(weight for _, weight in weighted)
weighted.insert(0, ("positive", baseline_weight))
result = None
for name, weight in weighted:
if weight == 0:
continue
prediction = comfy.samplers.sampling_function(
self.inner_model, x, timestep, None, self.conds[name], 1.0, model_options=model_options, seed=seed)
if result is None:
result = prediction if weight == 1 else prediction * weight
else:
result = result + prediction * weight
return result
def reference_cond_batch(args):
outputs = []
sigma = float(args["sigma"][0])
for conditioning in args["conds"]:
baseline, branches = [], {}
for item in conditioning or []:
marker = item.get("fl_krea_reference")
if marker is None:
baseline.append(item)
else:
branches.setdefault(marker, []).append(item)
if not branches:
outputs.append(comfy.samplers.calc_cond_batch(args["model"], [conditioning], args["input"], args["sigma"], args["model_options"])[0])
continue
weighted = [(cond, weight * sigma_envelope(sigma, bounds)) for (_, weight, bounds), cond in branches.items()]
weighted.insert(0, (baseline, 1.0 - math.fsum(weight for _, weight in weighted)))
result = None
for cond, weight in weighted:
if weight == 0:
continue
prediction = comfy.samplers.calc_cond_batch(args["model"], [cond], args["input"], args["sigma"], args["model_options"])[0]
if result is None:
result = prediction if weight == 1 else prediction * weight
else:
result = result + prediction * weight
outputs.append(result)
return outputs
class FL_KreaReference(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="FL_KreaReference", display_name="FL Krea Reference", category="Fill Nodes/Conditioning",
description="Define what to borrow from one Krea reference. Roles guide interpretation; they do not lock identity or geometry.",
inputs=[
io.Image.Input("image"),
io.Boolean.Input("enabled", default=True),
io.Combo.Input("role", options=list(ROLES), default="style"),
io.Float.Input("weight", default=1.0, min=0.0, max=1.0, step=0.05, tooltip="Reference strength: 0 is off, 0.05 applies 5%, and 1 applies the full contribution before overall influence. Multiple references add their contributions."),
io.Combo.Input("resolution", options=list(RESOLUTIONS), default=512, tooltip="Longest-side limit before vision encoding."),
io.Float.Input("start", default=0.0, min=0.0, max=1.0, step=0.01, advanced=True),
io.Float.Input("end", default=1.0, min=0.0, max=1.0, step=0.01, advanced=True),
io.Float.Input("fade", default=0.0, min=0.0, max=0.5, step=0.01, advanced=True, tooltip="Fraction of the window used to fade at each edge, interpolated in sigma space."),
io.Combo.Input("reference_mode", options=["context", "full"], default="context", advanced=True,
tooltip="Context keeps image-informed text states to reduce copied layouts. Full also supplies visual tokens for stronger source resemblance."),
], outputs=[Reference.Output(display_name="reference")])
@classmethod
def execute(cls, image, enabled=True, role="style", weight=1.0, resolution=512, start=0.0, end=1.0, fade=0.0, reference_mode="context"):
if image.ndim != 4 or image.shape[0] != 1 or image.shape[-1] != 3:
raise ValueError("Krea Reference needs one RGB image. Select one image from the batch first.")
if role not in ROLES or resolution not in RESOLUTIONS or reference_mode not in ("context", "full"):
raise ValueError("Krea Reference: choose a listed role, resolution, and reference mode.")
if not math.isfinite(weight) or not 0 <= weight <= 1:
raise ValueError("Krea Reference weight must be between 0 and 1.")
if not 0 <= start < end <= 1 or not 0 <= fade <= 0.5:
raise ValueError("Krea Reference needs 0 <= start < end <= 1 and fade between 0 and 0.5.")
return io.NodeOutput(dict(image=image, enabled=enabled, role=role, weight=weight,
resolution=resolution, start=start, end=end, fade=fade, reference_mode=reference_mode))
class FL_KreaReferenceGuider(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="FL_KreaReferenceGuider", display_name="FL Krea Reference Guider", category="Fill Nodes/Conditioning",
description="Blend independent Krea reference predictions. Use GUIDER with SamplerCustomAdvanced, or connect both MODEL and CONDITIONING to a KSampler (positive, CFG 1). Each active reference adds a model evaluation.",
inputs=[
io.Model.Input("model"),
io.Clip.Input("clip"),
io.String.Input("prompt", multiline=True, default="A dog in the park."),
io.Float.Input("influence", default=0.7, min=0.0, max=1.0, step=0.05, tooltip="Overall reference strength after blending. Zero uses only the scene prompt."),
io.Autogrow.Input("references", optional=True, template=io.Autogrow.TemplatePrefix(
input=Reference.Input("reference"), prefix="reference_", min=0, max=100)),
io.Combo.Input("blend_mode", options=["add", "average"], default="add", optional=True,
tooltip="Add sums reference contributions. Average blends toward dividing them by the number of enabled references."),
io.Float.Input("average_amount", default=1.0, min=0.0, max=1.0, step=0.05, optional=True,
display_mode=io.NumberDisplay.slider,
tooltip="In Average mode: 0 keeps the additive blend; 1 fully averages enabled references. Individual weights still apply. Fading or zero-weight references keep their share; disabled references are excluded."),
], outputs=[io.Guider.Output(), io.Model.Output(), io.Conditioning.Output()])
@classmethod
def execute(cls, model, clip, prompt, influence=0.7, references=None, blend_mode="add", average_amount=1.0):
if not isinstance(model.model, comfy.model_base.Krea2) or not isinstance(clip.tokenizer, Krea2Tokenizer):
raise ValueError("Use a Krea 2 model and CLIPLoader with type 'krea2'.")
if not 0 <= influence <= 1:
raise ValueError("Krea reference influence must be between 0 and 1.")
if blend_mode not in ("add", "average"):
raise ValueError("Krea Reference Guider: choose add or average.")
if not 0 <= average_amount <= 1:
raise ValueError("Krea reference average amount must be between 0 and 1.")
if any(key in model.model_options for key in ("sampler_cfg_function", "sampler_pre_cfg_function", "sampler_post_cfg_function", "sampler_calc_cond_batch_function")):
raise ValueError("Krea Reference Guider uses CFG 1. Remove custom CFG/guidance patches from its model input.")
baseline = clip.encode_from_tokens_scheduled(clip.tokenize(prompt))
enabled_references = [reference for reference in (references or {}).values() if reference["enabled"]]
if blend_mode == "average" and enabled_references:
influence *= 1.0 - average_amount + average_amount / len(enabled_references)
branches = []
if influence > 0:
for reference in enabled_references:
if reference["weight"] > 0:
branches.append((reference, encode_reference(clip, prompt, reference)))
guider = KreaReferenceGuider(model, baseline, branches, influence)
conditioning = list(baseline)
for (name, weight, bounds), (_, encoded) in zip(guider.references, branches):
for tensor, metadata in encoded:
conditioning.append([tensor, {**metadata, "fl_krea_reference": (name, influence * weight, bounds)}])
sampler_model = model.clone()
sampler_model.set_model_sampler_calc_cond_batch_function(reference_cond_batch)
return io.NodeOutput(guider, sampler_model, conditioning)
+2 -2
View File
@@ -127,8 +127,6 @@ def _parse_settings(load_settings, width_override=None, height_override=None):
raise ValueError("FL Load Image width and height overrides must be between 0 and 16384.")
settings[name] = value
if resize_mode == "fit" and settings["width"] == 0 and settings["height"] == 0:
raise ValueError("FL Load Image fit resize requires a width or height.")
if resize_mode == "crop" and (settings["width"] == 0 or settings["height"] == 0):
raise ValueError("FL Load Image crop resize requires both width and height.")
return settings
@@ -143,6 +141,8 @@ def _target_dimensions(source_width, source_height, settings):
width = settings["width"]
height = settings["height"]
if width == 0 and height == 0:
return source_width, source_height
if width == 0:
scale = height / source_height
elif height == 0:
+6 -3
View File
@@ -2,6 +2,7 @@ import copy
import torch
import math
from nodes import common_ksampler, VAEDecode, VAEEncode
import comfy.sample
import comfy.samplers
import comfy.utils
import logging
@@ -170,8 +171,10 @@ class FL_KsamplerPlus:
if use_sliced_conditioning:
batch_size = 1
# Handle variable tensor dimensions (4D or 5D)
latent_samples = latent_image["samples"]
# Resolve the model's native latent layout before sizing the tiles and canvas.
latent_samples = comfy.sample.fix_empty_latent_channels(
model, latent_image["samples"], latent_image.get("downscale_ratio_spacial"),
latent_image.get("downscale_ratio_temporal"))
primary_samples = primary_tensor(latent_samples)
if latent_samples.is_nested:
batch_size = 1
@@ -228,7 +231,7 @@ class FL_KsamplerPlus:
if primary_noise_mask is not None and is_video:
if primary_noise_mask.shape[-1] > 1 and primary_noise_mask.shape[-2] > 1:
# Spatially varying mask — slice to match tile
sliced_noise_mask = primary_noise_mask[:, :, :, y_start:y_end, x_start:x_end]
sliced_noise_mask = primary_noise_mask[..., y_start:y_end, x_start:x_end]
else:
# Spatially uniform (e.g., [B,1,T,1,1]) — pass as-is
sliced_noise_mask = primary_noise_mask
+133 -70
View File
@@ -14,16 +14,21 @@
# The soft masks come from FL_KsamplerSEG_Regions; they overlap and feather
# already, so coverage is guaranteed.
import base64
import io
import logging
import math
import torch
import torch.nn.functional as F
from PIL import Image
import comfy.sample
import comfy.samplers
import comfy.utils
import comfy.model_management
import latent_preview
from server import PromptServer
from .FL_KsamplerSEG_common import unwrap_regions, latent_bbox_from_image_bbox
from ._latent_helpers import primary_only_noise_mask, primary_tensor, replace_primary_tensor
@@ -38,7 +43,7 @@ class FL_KsamplerSEG:
return {
"required": {
"model": ("MODEL",),
"regions": ("SEG_REGIONS",),
"regions": ("SEG_REGIONS", {"tooltip": "Encoded regions override positive and negative. Use raw Regions to preserve connected reference conditioning."}),
"latent_image": ("LATENT",),
"positive": ("CONDITIONING",),
"negative": ("CONDITIONING",),
@@ -50,6 +55,7 @@ class FL_KsamplerSEG:
"denoise": ("FLOAT", {"default": 0.55, "min": 0.0, "max": 1.0, "step": 0.01}),
"cascade_start_corner": (CASCADE_START_CORNERS, {"default": "top_left"}),
},
"hidden": {"unique_id": "UNIQUE_ID"},
}
RETURN_TYPES = ("LATENT",)
@@ -63,8 +69,21 @@ class FL_KsamplerSEG:
def sample(self, model, regions, latent_image, positive, negative,
seed, steps, cfg, sampler_name, scheduler, denoise,
cascade_start_corner):
cascade_start_corner, unique_id=None):
return self._sample_regions(model, regions, latent_image, positive, negative,
seed, steps, cfg, sampler_name, scheduler, denoise,
cascade_start_corner, unique_id)
def _sample_regions(self, model, regions, latent_image, positive, negative,
seed, steps, cfg, sampler_name, scheduler, denoise,
cascade_start_corner, unique_id=None, *, disable_noise=False,
start_step=None, last_step=None, force_full_denoise=False):
regions = unwrap_regions(regions)
window_start = 0 if start_step is None else start_step
window_end = steps if last_step is None else min(steps, last_step)
region_steps = max(0, window_end - window_start)
if region_steps == 0:
return (latent_image,)
latent_samples = latent_image["samples"]
primary_samples = primary_tensor(latent_samples)
@@ -77,34 +96,15 @@ class FL_KsamplerSEG:
"An empty latent at low denoise produces noise."
)
H, W = regions["image_size"]
latent_h = primary_samples.shape[-2]
latent_w = primary_samples.shape[-1]
# Auto-detect the actual spatial downscale from the model. Different
# models use different ratios -- SD/SDXL=8, Flux=8, LTX-Video=32,
# Hunyuan-Video=16, etc. Use the model's authoritative value rather
# than the user's hint on the Regions node, which won't know.
downscale = self._resolve_downscale(model, regions, latent_h, latent_w, H, W)
expected_lh = (H + downscale - 1) // downscale
expected_lw = (W + downscale - 1) // downscale
if abs(latent_h - expected_lh) > 1 or abs(latent_w - expected_lw) > 1:
logging.warning(
f"[FL_KsamplerSEG] latent shape ({latent_h}x{latent_w}) doesn't match "
f"regions image_size {H}x{W} / downscale={downscale}. Sampling may misalign."
)
# One-line diagnostic so users can confirm the right downscale was picked.
print(f"[FL_KsamplerSEG] latent={tuple(primary_samples.shape)} "
f"regions={H}x{W} downscale={downscale}")
device = model.load_device
latent_full = comfy.sample.fix_empty_latent_channels(
model, latent_samples.to(device=device),
latent_image.get("downscale_ratio_spacial", None),
)
H, W = regions["image_size"]
latent_h, latent_w = primary_tensor(latent_full).shape[-2:]
downscale = self._resolve_downscale(model, regions, latent_h, latent_w, H, W)
N = regions["shape_masks"].shape[0]
per_region_cond = regions.get("conditioning_per_region")
@@ -150,7 +150,23 @@ class FL_KsamplerSEG:
# it must broadcast as (1,1,1,H,W) -- one extra leading dim per
# non-spatial axis. PyTorch won't auto-align mismatched-rank tensors.
latent_ndim = canvas.ndim
for spec in region_specs:
total_steps = region_steps * len(region_specs)
preview_callback = latent_preview.prepare_callback(model, total_steps)
for region_step, spec in enumerate(region_specs):
preview_state = {
"node": str(unique_id), "region": spec["region_index"],
"position": region_step + 1, "count": len(region_specs),
"step": 0, "steps": region_steps, "state": "sampling",
"start_step": start_step, "end_step": window_end, "schedule_steps": steps,
"crop": list(spec["latent_bbox"]), "size": [latent_w, latent_h],
"mask": self._preview_mask(spec["write_lat"]),
}
def send_status():
if unique_id is not None and PromptServer.instance is not None:
PromptServer.instance.send_sync("fl_seg_sampling", dict(preview_state), PromptServer.instance.client_id)
send_status()
samples = self._sample_one_region_full(
model=model,
source_latent=replace_primary_tensor(latent_full, canvas),
@@ -158,6 +174,10 @@ class FL_KsamplerSEG:
steps=steps, cfg=cfg, sampler_name=sampler_name,
scheduler=scheduler, denoise=denoise,
latent_ndim=latent_ndim,
preview_callback=preview_callback, step_offset=region_step * region_steps,
total_steps=total_steps, preview_state=preview_state, send_status=send_status,
disable_noise=disable_noise, start_step=start_step, last_step=last_step,
force_full_denoise=force_full_denoise,
)
by0, bx0, by1, bx1 = spec["latent_bbox"]
comp_b = self._reshape_mask_for_broadcast(
@@ -167,6 +187,9 @@ class FL_KsamplerSEG:
existing = canvas[..., by0:by1, bx0:bx1]
canvas[..., by0:by1, bx0:bx1] = samples_dev * comp_b + existing * (1.0 - comp_b)
preview_state["state"] = "complete"
send_status()
out = replace_primary_tensor(latent_full, canvas).to(
device=comfy.model_management.intermediate_device(),
dtype=comfy.model_management.intermediate_dtype(),
@@ -238,7 +261,10 @@ class FL_KsamplerSEG:
def _sample_one_region_full(self, *, model, source_latent, spec,
steps, cfg, sampler_name, scheduler, denoise,
latent_ndim=None):
latent_ndim=None, preview_callback=None, step_offset=0,
total_steps=None, preview_state=None, send_status=None,
disable_noise=False, start_step=None, last_step=None,
force_full_denoise=False):
"""Run a complete sampler call (all steps) for one region, sourcing the
crop from `source_latent` (the in-progress canvas)."""
by0, bx0, by1, bx1 = spec["latent_bbox"]
@@ -246,7 +272,8 @@ class FL_KsamplerSEG:
primary_crop = source_primary[..., by0:by1, bx0:bx1].contiguous()
latent_crop = replace_primary_tensor(source_latent, primary_crop)
noise = comfy.sample.prepare_noise(latent_crop.cpu(), spec["seed"])
noise = (comfy.sample.prepare_empty_noise(latent_crop) if disable_noise
else comfy.sample.prepare_noise(latent_crop.cpu(), spec["seed"]))
# Mask must match the latent rank so it broadcasts. For 4D latent
# (image): (1,1,H,W). For 5D latent (video): (1,1,1,H,W).
@@ -257,16 +284,44 @@ class FL_KsamplerSEG:
).to(dtype=primary_crop.dtype)
noise_mask = primary_only_noise_mask(latent_crop, noise_mask)
callback = latent_preview.prepare_callback(model, steps)
window_start = 0 if start_step is None else start_step
window_end = steps if last_step is None else min(steps, last_step)
region_steps = max(0, window_end - window_start)
if preview_callback is None:
preview_callback = latent_preview.prepare_callback(model, region_steps)
if total_steps is None:
total_steps = region_steps
preview_canvas = None
preview_base = None
def callback(step, x0, x, _total_steps):
nonlocal preview_canvas, preview_base
if preview_canvas is None:
preview_canvas = model.get_model_object("process_latent_in")(source_primary).clone()
preview_base = preview_canvas[..., by0:by1, bx0:bx1].clone()
comp = self._reshape_mask_for_broadcast(spec["comp_lat"], preview_canvas.ndim)
preview_canvas[..., by0:by1, bx0:bx1] = primary_tensor(x0) * comp + preview_base * (1.0 - comp)
# Some samplers emit one more preview after the final diffusion step.
completed_steps = min(step + 1, region_steps)
preview_callback(step_offset + completed_steps - 1, preview_canvas, x, total_steps)
if preview_state is not None:
preview_state["step"] = completed_steps
preview_state.pop("mask", None)
send_status()
try:
samples = comfy.sample.sample(
model, noise, steps, cfg, sampler_name, scheduler,
spec["pos_cond"], spec["neg_cond"], latent_crop,
denoise=denoise, noise_mask=noise_mask,
disable_noise=disable_noise, start_step=start_step, last_step=last_step,
force_full_denoise=force_full_denoise,
callback=callback, disable_pbar=True, seed=spec["seed"],
)
except Exception as e:
if preview_state is not None:
preview_state["state"] = "stopped"
send_status()
logging.error(
f"[FL_KsamplerSEG] region {spec['region_index']} sample failed: {e}"
)
@@ -274,6 +329,15 @@ class FL_KsamplerSEG:
return primary_tensor(samples)
@staticmethod
def _preview_mask(mask):
mask = mask.detach().float().cpu()
image = Image.fromarray((mask.clamp(0, 1).numpy() * 255).astype("uint8"))
image.thumbnail((160, 160))
buffer = io.BytesIO()
image.save(buffer, format="PNG")
return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii")
@staticmethod
def _cascade_order(*, regions, write_areas, start_corner):
"""Corner-anchored nearest-neighbor traversal of regions for cascade_paint.
@@ -344,48 +408,15 @@ class FL_KsamplerSEG:
@staticmethod
def _resolve_downscale(model, regions, latent_h, latent_w, image_h, image_w):
"""Determine the actual spatial downscale ratio.
Order of preference:
1. The model's latent_format.spacial_downscale_ratio if it matches
the observed latent shape (this is the authoritative source --
SD/SDXL=8, Flux=8, LTX-Video=32, Hunyuan-Video=16, Wan=16, etc.)
2. Inferred from latent_shape / image_shape if (1) doesn't match
3. The regions dict's downscale_ratio hint as a final fallback
"""
regions_hint = int(regions.get("downscale_ratio", 8))
# Try the model's authoritative value.
try:
latent_format = model.get_model_object("latent_format")
model_ratio = int(getattr(latent_format, "spacial_downscale_ratio", 8))
# Verify it matches the observed shape. Allow ±1 for rounding.
expected_lh = (image_h + model_ratio - 1) // model_ratio
expected_lw = (image_w + model_ratio - 1) // model_ratio
if abs(latent_h - expected_lh) <= 1 and abs(latent_w - expected_lw) <= 1:
if model_ratio != regions_hint:
print(
f"[FL_KsamplerSEG] model uses downscale={model_ratio} "
f"(regions hint was {regions_hint}); using model's value."
)
return model_ratio
except Exception:
pass
# Infer from observed shapes.
if latent_h > 0 and latent_w > 0:
inferred_h = round(image_h / latent_h)
inferred_w = round(image_w / latent_w)
if inferred_h == inferred_w and inferred_h >= 1:
if inferred_h != regions_hint:
print(
f"[FL_KsamplerSEG] inferred downscale={inferred_h} from "
f"latent vs image dims (regions hint was {regions_hint})."
)
return inferred_h
# Last resort.
return regions_hint
downscale = int(model.get_model_object("latent_format").spacial_downscale_ratio)
if not (image_h // downscale <= latent_h <= math.ceil(image_h / downscale)
and image_w // downscale <= latent_w <= math.ceil(image_w / downscale)):
raise ValueError(
f"SEG Regions describe {image_w}x{image_h}, but the latent represents "
f"{latent_w * downscale}x{latent_h * downscale}. "
"Connect Regions and VAE Encode to the same resized image."
)
return downscale
@staticmethod
def _reshape_mask_for_broadcast(mask_2d, target_ndim):
@@ -412,3 +443,35 @@ class FL_KsamplerSEG:
pad_h = max(0, h - ch)
pad_w = max(0, w - cw)
return F.pad(t, (0, pad_w, 0, pad_h), mode="constant", value=0.0)
class FL_KsamplerSEGAdvanced(FL_KsamplerSEG):
@classmethod
def INPUT_TYPES(cls):
inputs = super().INPUT_TYPES()
required = inputs["required"]
seed = required.pop("seed")
required.pop("denoise")
corner = required.pop("cascade_start_corner")
inputs["required"] = {
**{name: required[name] for name in ("model", "regions", "latent_image", "positive", "negative")},
"add_noise": (["enable", "disable"], {"default": "enable", "tooltip": "Enable for a source latent. Disable when continuing a latent that already contains noise."}),
"noise_seed": (seed[0], {**seed[1], "control_after_generate": True}),
**{name: required[name] for name in ("steps", "cfg", "sampler_name", "scheduler")},
"start_at_step": ("INT", {"default": 0, "min": 0, "max": 10000, "tooltip": "Start index in the full schedule, applied to every region. Zero starts at the beginning."}),
"end_at_step": ("INT", {"default": 10000, "min": 0, "max": 10000, "tooltip": "Stop index in the full schedule. Values above steps run to the end. An empty step window returns the input unchanged."}),
"return_with_leftover_noise": (["disable", "enable"], {"default": "disable", "tooltip": "Disable to finish at zero noise even when stopping early. Enable to retain noise for a following sampler."}),
"cascade_start_corner": corner,
}
return inputs
DESCRIPTION = "Sample each SEG region over an explicit window of the full diffusion schedule. Uses KSampler Advanced noise and end-step behavior. Overlapping regions are composited in cascade order, so splitting across nodes is not identical to one uninterrupted pass."
def sample(self, model, regions, latent_image, positive, negative, add_noise,
noise_seed, steps, cfg, sampler_name, scheduler, start_at_step,
end_at_step, return_with_leftover_noise, cascade_start_corner, unique_id=None):
return self._sample_regions(
model, regions, latent_image, positive, negative, noise_seed, steps, cfg,
sampler_name, scheduler, 1.0, cascade_start_corner, unique_id,
disable_noise=add_noise == "disable", start_step=start_at_step,
last_step=end_at_step, force_full_denoise=return_with_leftover_noise == "disable")
+59
View File
@@ -0,0 +1,59 @@
import comfy.model_base
from comfy.text_encoders.krea2 import Krea2Tokenizer
from ..conditioning.FL_KreaReference import FL_KreaReference, KreaReferenceGuider, encode_reference, reference_cond_batch
from .FL_KsamplerSEG_common import attach_conditioning, latent_bbox_from_image_bbox, unwrap_regions
class FL_KsamplerSEG_Krea:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"regions": ("SEG_REGIONS",),
"source_image": ("IMAGE",),
"prompt": ("STRING", {"multiline": True, "default": "Enhance natural detail while preserving the reference crop's content, composition, lighting and colors. Do not add objects or extend the scene."}),
"reference_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05, "tooltip": "0 uses the prompt alone; 1 uses the matching source crop. Intermediate values blend both predictions and require an extra model evaluation."}),
"reference_resolution": ([256, 512, 1024, 1280], {"default": 512}),
}}
RETURN_TYPES = ("MODEL", "SEG_REGIONS", "CONDITIONING")
RETURN_NAMES = ("model", "regions", "conditioning")
FUNCTION = "encode"
CATEGORY = "Fill Nodes/Ksamplers"
DESCRIPTION = "Encode each SEG sampler crop as its own Krea 2 visual reference. Connect the same resized source used by Regions and VAE Encode. Use all three outputs with FL Ksampler SEG at CFG 1."
def encode(self, model, clip, regions, source_image, prompt, reference_strength=1.0, reference_resolution=512):
regions = unwrap_regions(regions)
if not isinstance(model.model, comfy.model_base.Krea2) or not isinstance(clip.tokenizer, Krea2Tokenizer):
raise ValueError("Use a Krea 2 model and CLIPLoader with type 'krea2'.")
if any(key in model.model_options for key in ("sampler_cfg_function", "sampler_pre_cfg_function", "sampler_post_cfg_function", "sampler_calc_cond_batch_function")):
raise ValueError("Krea region references use CFG 1. Remove custom CFG/guidance patches from the model input.")
if not 0 <= reference_strength <= 1:
raise ValueError("Krea region reference strength must be between 0 and 1.")
height, width = regions["image_size"]
if source_image.ndim != 4 or source_image.shape[0] != 1 or source_image.shape[-1] != 3 or tuple(source_image.shape[1:3]) != (height, width):
raise ValueError("Krea region references need one RGB source image at the same size as Regions. Connect the same resized image to both nodes and VAE Encode.")
downscale = int(model.get_model_object("latent_format").spacial_downscale_ratio)
if height % downscale or width % downscale:
raise ValueError(f"Resize the source to multiples of {downscale} pixels so Krea references align with the VAE latent crops.")
baseline = clip.encode_from_tokens_scheduled(clip.tokenize(prompt))
per_region = []
for bbox in regions["padded_bboxes"]:
conditioning = list(baseline)
if reference_strength > 0:
y0, x0, y1, x1 = latent_bbox_from_image_bbox(bbox, downscale, height // downscale, width // downscale)
crop = source_image[:, y0 * downscale:min(height, y1 * downscale), x0 * downscale:min(width, x1 * downscale)]
reference = FL_KreaReference.execute(
crop, role="custom",
weight=reference_strength, resolution=reference_resolution, reference_mode="full").result[0]
encoded = encode_reference(clip, prompt, reference)
guider = KreaReferenceGuider(model, baseline, [(reference, encoded)], 1.0)
name, weight, bounds = guider.references[0]
for tensor, metadata in encoded:
conditioning.append([tensor, {**metadata, "fl_krea_reference": (name, weight, bounds)}])
per_region.append((conditioning, baseline))
sampler_model = model.clone()
sampler_model.set_model_sampler_calc_cond_batch_function(reference_cond_batch)
return sampler_model, attach_conditioning(regions, per_region), baseline
+71 -52
View File
@@ -1,21 +1,3 @@
# FL_KsamplerSEG_Regions: tessellate an image into N Voronoi cells with
# overlapping, feathered region masks so EVERY pixel is touched by at least
# one diffusion pass and seams blend smoothly. Optional Lloyd relaxation for
# uniform cell sizes; optional safe-zone subtraction.
#
# Mask design (post-fix):
# - One soft mask per region. Built by dilating the hard Voronoi cell, then
# gaussian-blurring the edge. The result has values in [0, 1] that taper
# across the boundary into adjacent cells.
# - The SAME soft mask is used as both the diffusion noise_mask AND the
# composite blend mask. This guarantees: (a) the model partially updates
# pixels in the overlap band, (b) the alpha-accumulator math is consistent
# and normalizes to ~1.0 everywhere, (c) the union of all masks covers the
# entire canvas (modulo safe_zone).
#
# Performance: Voronoi labeling, mask processing, and Lloyd relaxation all run
# on the user's torch device (GPU when available) using batched ops.
import io
import base64
import math
@@ -24,6 +6,7 @@ import torch
import torch.nn.functional as F
import numpy as np
from PIL import Image, ImageDraw, ImageFont
from scipy.ndimage import distance_transform_edt
from server import PromptServer
@@ -44,12 +27,16 @@ class FL_KsamplerSEG_Regions:
"downscale_ratio": ("INT", {"default": 8, "min": 1, "max": 16, "step": 1}),
"seed": ("INT", {"default": 0, "min": 0, "max": 0x7fffffff}),
"show_preview": ("BOOLEAN", {"default": True}),
"preview_mode": (["overlay", "coverage_heatmap"], {"default": "overlay"}),
"preview_mode": (["overlay", "coverage_heatmap", "sampler_crops"], {"default": "overlay"}),
},
"optional": {
"image": ("IMAGE",),
"latent": ("LATENT",),
"safe_zone_mask": ("MASK",),
"margin_mode": (["pixels", "legacy"], {"default": "pixels", "tooltip": "Bounded margins in output-image pixels, quantized to the downscale grid. Legacy preserves factor-based workflows."}),
"overlap_width_px": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 8, "tooltip": "Total shared band across a cell boundary. Each neighboring mask extends by half this distance. Zero means no expansion."}),
"feather_width_px": ("INT", {"default": 32, "min": 0, "max": 2048, "step": 8, "tooltip": "Fade inside the expanded edge; never enlarges the region. Cannot exceed half the overlap width."}),
"context_padding_px": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 8, "tooltip": "Extra image context per crop edge, outside the edit mask. Does not expand the edit mask."}),
},
"hidden": {"unique_id": "UNIQUE_ID"},
}
@@ -62,7 +49,12 @@ class FL_KsamplerSEG_Regions:
def build(self, num_regions, relaxation_iterations, region_overlap_factor,
edge_softness, context_padding_factor, safe_zone_feather_px,
downscale_ratio, seed, show_preview, preview_mode,
image=None, latent=None, safe_zone_mask=None, unique_id=None):
image=None, latent=None, safe_zone_mask=None, unique_id=None,
margin_mode="legacy", overlap_width_px=64, feather_width_px=32, context_padding_px=64):
if margin_mode not in ("pixels", "legacy"):
raise ValueError("Choose pixels or legacy for margin_mode.")
if margin_mode == "pixels" and not (0 <= feather_width_px <= overlap_width_px / 2 and context_padding_px >= 0):
raise ValueError("Feather width must be between zero and half the overlap width; context padding must be non-negative.")
if image is None and latent is None:
raise ValueError(
"FL_KsamplerSEG_Regions: connect either 'image' or 'latent' to size the regions."
@@ -83,12 +75,8 @@ class FL_KsamplerSEG_Regions:
device = self._pick_device()
# All heavy mask work runs at "work resolution" -- 2x latent res. The
# sampler downsamples masks 8x to latent space anyway, so doing them
# at 8x* of full means we redundantly compute ~64x more pixels than
# needed. work_scale = downscale_ratio * 2 keeps a 2x margin for
# crisp boundaries.
work_scale = max(1, int(downscale_ratio) * 2)
# Pixel margins use the latent grid; legacy keeps its original coarser grid.
work_scale = max(1, int(downscale_ratio) * (2 if margin_mode == "legacy" else 1))
H_w = max(8, (H + work_scale - 1) // work_scale)
W_w = max(8, (W + work_scale - 1) // work_scale)
sx = W_w / float(W)
@@ -120,25 +108,18 @@ class FL_KsamplerSEG_Regions:
)
hard_masks_w = hard_masks_w[non_empty]
# Per-cell typical size in WORK-res pixels.
cell_areas_w = hard_masks_w.flatten(1).sum(dim=1)
avg_cell_side_w = float(cell_areas_w.mean().sqrt().item())
# Dilation + feather radii in work-res pixels (proportionally smaller
# than the full-res equivalents -- this is where the speedup comes from).
dilate_w = max(1.0, float(region_overlap_factor) * avg_cell_side_w)
feather_w = max(1.0, float(edge_softness) * avg_cell_side_w)
# Build soft masks at work res. Both ops batched + grouped so N regions
# cost roughly the same as 1.
soft_masks_w = self._dilate_then_blur(hard_masks_w, dilate_w, feather_w)
# Per-region renormalize so each soft mask peaks at 1.0.
peaks = soft_masks_w.flatten(1).amax(dim=1).clamp(min=1e-6).view(-1, 1, 1)
soft_masks_w = (soft_masks_w / peaks).clamp(0.0, 1.0)
# Bbox extraction at WORK res, vectorized via row/col any (no Python loop).
bboxes_w = self._extract_bboxes_vectorized(soft_masks_w, alpha_threshold=0.01)
if margin_mode == "pixels":
soft_masks_w = self._pixel_masks(hard_masks_w, overlap_width_px / 2, feather_width_px, H / H_w, W / W_w)
bboxes_w = self._extract_bboxes_vectorized(hard_masks_w)
else:
cell_areas_w = hard_masks_w.flatten(1).sum(dim=1)
avg_cell_side_w = float(cell_areas_w.mean().sqrt().item())
dilate_w = max(1.0, float(region_overlap_factor) * avg_cell_side_w)
feather_w = max(1.0, float(edge_softness) * avg_cell_side_w)
soft_masks_w = self._dilate_then_blur(hard_masks_w, dilate_w, feather_w)
peaks = soft_masks_w.flatten(1).amax(dim=1).clamp(min=1e-6).view(-1, 1, 1)
soft_masks_w = (soft_masks_w / peaks).clamp(0.0, 1.0)
bboxes_w = self._extract_bboxes_vectorized(soft_masks_w, alpha_threshold=0.01)
# Scale work-res bboxes back up to image-res. Round outward so we never
# truncate the soft mask at the bbox edge.
@@ -154,7 +135,7 @@ class FL_KsamplerSEG_Regions:
padded_bboxes = []
for (y0, x0, y1, x1) in bboxes:
max_side = max(y1 - y0, x1 - x0)
pad = int(round(float(context_padding_factor) * max_side))
pad = math.ceil(overlap_width_px / 2 + context_padding_px) if margin_mode == "pixels" else int(round(float(context_padding_factor) * max_side))
padded_bboxes.append((
max(0, y0 - pad),
max(0, x0 - pad),
@@ -190,16 +171,15 @@ class FL_KsamplerSEG_Regions:
if safe_zone_mask is None and min_cov < 0.01:
print(
f"[FL_KsamplerSEG_Regions] warning: coverage minimum is {min_cov:.4f}; "
f"some pixels may not be diffused. Increase region_overlap_factor."
f"some pixels may not be diffused. Check region masks."
)
# Single full-res upsample of all soft masks at the end. Bilinear is
# fine because the masks are already smooth (gaussian-blurred) at
# work res; upsampling adds no aliasing artifacts.
# Preserve the pixel-mode latent grid without adding another blur band.
if (H_w, W_w) != (H, W):
soft_masks = F.interpolate(
soft_masks_w.unsqueeze(0), size=(H, W),
mode="bilinear", align_corners=False,
mode="nearest" if margin_mode == "pixels" else "bilinear",
align_corners=None if margin_mode == "pixels" else False,
).squeeze(0).clamp(0.0, 1.0)
else:
soft_masks = soft_masks_w
@@ -223,7 +203,9 @@ class FL_KsamplerSEG_Regions:
# Visualization. Upsample the work-res coverage / labels to full res
# for display. (The visualization is the only place we touch full res
# outside of the final mask upsample.)
if preview_mode == "coverage_heatmap":
if preview_mode == "sampler_crops":
viz = self._build_crop_viz(viz_source, shape_t, padded_bboxes)
elif preview_mode == "coverage_heatmap":
coverage_full = F.interpolate(
coverage_w.unsqueeze(0).unsqueeze(0), size=(H, W),
mode="bilinear", align_corners=False,
@@ -252,6 +234,10 @@ class FL_KsamplerSEG_Regions:
"size": [W, H],
"min_coverage": round(min_cov, 4),
"max_coverage": round(max_cov, 4),
"mode": preview_mode,
"margins": None if margin_mode == "legacy" else {
"overlap": overlap_width_px, "feather": feather_width_px, "context": context_padding_px,
},
},
)
except Exception as e:
@@ -574,3 +560,36 @@ class FL_KsamplerSEG_Regions:
buf = io.BytesIO()
pil.save(buf, format="PNG")
return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
@staticmethod
def _build_crop_viz(image_np, masks, bboxes):
source = Image.fromarray(image_np)
sheet = Image.new("RGB", (640, 350 * math.ceil(len(bboxes) / 2)), "#18181b")
draw = ImageDraw.Draw(sheet)
for i, (y0, x0, y1, x1) in enumerate(bboxes):
crop = source.crop((x0, y0, x1, y1))
crop.thumbnail((304, 310), Image.Resampling.LANCZOS)
mask = Image.fromarray((masks[i, y0:y1, x0:x1].numpy() * 255).astype(np.uint8))
mask = mask.resize(crop.size, Image.Resampling.BILINEAR)
context = Image.blend(crop, Image.new("RGB", crop.size, "#06131a"), 0.7)
crop = Image.composite(crop, context, mask)
x, y = (i % 2) * 320, (i // 2) * 350
draw.text((x + 8, y + 8), f"Mask {i}: {x1-x0} x {y1-y0}", fill="white")
sheet.paste(crop, (x + (320 - crop.width) // 2, y + 32))
return np.array(sheet)
@staticmethod
def _pixel_masks(hard_masks, expansion_px, feather_px, pixel_h, pixel_w):
if expansion_px == 0:
return hard_masks
masks = []
for hard in hard_masks.detach().cpu().numpy().astype(bool):
distance = distance_transform_edt(~hard, sampling=(pixel_h, pixel_w))
distance = np.maximum(0, distance - min(pixel_h, pixel_w) / 2)
if feather_px == 0:
soft = (distance <= expansion_px).astype(np.float32)
else:
soft = np.clip((expansion_px - distance) / feather_px, 0, 1).astype(np.float32)
soft[hard] = 1
masks.append(torch.from_numpy(soft))
return torch.stack(masks).to(device=hard_masks.device)
+125
View File
@@ -0,0 +1,125 @@
import json
import logging
import os
import torch
import comfy.lora
import comfy.model_management
import comfy.model_patcher
import comfy.utils
import folder_paths
def materialize_weight(model, key, device):
weight, _, convert = comfy.model_patcher.get_key_weight(model.model, key)
if key in model.backup:
weight = model.backup[key].weight
if key in model.hook_backup:
weight = model.hook_backup[key][0]
weight = comfy.model_management.cast_to_device(weight.detach(), device, torch.float32, copy=True)
if convert is not None:
weight = convert(weight, inplace=True)
return comfy.lora.calculate_weight(model.patches.get(key, []), weight, key)
def factorize_difference(diff, rank):
matrix = diff.flatten(1)
rank = min(rank, *matrix.shape)
if rank == min(matrix.shape):
up, values, down = torch.linalg.svd(matrix, full_matrices=False)
else:
up, values, right = torch.svd_lowrank(matrix, q=min(rank + 16, *matrix.shape), niter=4)
down = right.T
up = up[:, :rank] * values[:rank]
down = down[:rank]
if diff.ndim == 4:
up = up.reshape(diff.shape[0], rank, 1, 1)
down = down.reshape(rank, *diff.shape[1:])
return up, down
class FL_ModelDifferenceLoraSave:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"finetuned_model": ("MODEL", {"tooltip": "The fine-tune or merged model to extract."}),
"base_model": ("MODEL", {"tooltip": "The original model. Difference = fine-tune minus base."}),
"filename_prefix": ("STRING", {"default": "Krea2/Dirtyrealism_difference"}),
"rank": ("INT", {"default": 128, "min": 1, "max": 4096}),
"device": (["auto", "cpu"],),
}}
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("lora_path",)
FUNCTION = "save"
CATEGORY = "🏵️Fill Nodes/Model"
OUTPUT_NODE = True
DESCRIPTION = "Extract fine-tune minus base as a LoRA in the configured LoRA folder. Higher rank preserves more detail. FP8 sources include quantization error."
def save(self, finetuned_model, base_model, filename_prefix, rank, device):
if not 1 <= rank <= 4096:
raise ValueError("LoRA rank must be between 1 and 4096.")
if device not in ("auto", "cpu"):
raise ValueError("Choose auto or cpu for extraction.")
root = folder_paths.get_folder_paths("loras")[0]
if not filename_prefix.strip() or any(c in filename_prefix for c in ':*?"<>|\x00'):
raise ValueError("Use a relative LoRA filename prefix, such as Krea2/Dirtyrealism_difference.")
if not folder_paths.is_within_directory(root, os.path.join(root, filename_prefix)):
raise ValueError("The LoRA filename must stay inside the LoRA folder.")
directory, name, counter, _, _ = folder_paths.get_save_image_path(filename_prefix, root)
path = os.path.join(directory, f"{name}_{counter:05}_.safetensors")
if not folder_paths.is_within_directory(root, path):
raise ValueError("The LoRA filename must stay inside the LoRA folder.")
tuned = dict(finetuned_model.model.named_parameters())
base = dict(base_model.model.named_parameters())
keys = [k for k in tuned if k.startswith("diffusion_model.")]
base_keys = {k for k in base if k.startswith("diffusion_model.")}
if not keys or set(keys) != base_keys:
raise ValueError("The fine-tune and base must have the same diffusion model parameters.")
for key in keys:
if tuned[key].shape != base[key].shape:
raise ValueError(f"Model shape mismatch for {key}: {tuned[key].shape} vs {base[key].shape}.")
compute_device = comfy.model_management.get_torch_device() if device == "auto" else torch.device("cpu")
output = {}
progress = comfy.utils.ProgressBar(len(keys))
total_energy = 0.0
captured_energy = 0.0
for index, key in enumerate(keys):
comfy.model_management.throw_exception_if_processing_interrupted()
diff = materialize_weight(finetuned_model, key, compute_device)
diff.sub_(materialize_weight(base_model, key, compute_device))
if not torch.isfinite(diff).all():
raise ValueError(f"Non-finite model difference in {key}.")
energy = diff.square().sum().item()
total_energy += energy
if energy:
lora_key = key[:-7] if key.endswith(".weight") else key
if key.endswith(".weight") and diff.ndim in (2, 4):
up, down = factorize_difference(diff, rank)
captured_energy += up.square().sum().item()
output[lora_key + ".lora_up.weight"] = up.to(device="cpu", dtype=torch.float16, copy=True).contiguous()
output[lora_key + ".lora_down.weight"] = down.to(device="cpu", dtype=torch.float16, copy=True).contiguous()
del up, down
else:
output[lora_key + ".diff"] = diff.to(device="cpu", dtype=torch.float16, copy=True).contiguous()
captured_energy += energy
del diff
progress.update(1)
if (index + 1) % 16 == 0:
logging.info("FL LoRA extraction: %s/%s parameters", index + 1, len(keys))
if not output:
raise ValueError("The models have no weight differences; no LoRA was saved.")
if not all(torch.isfinite(t).all() for t in output.values()):
raise ValueError("The extracted difference exceeds FP16 range; no LoRA was saved.")
retained = captured_energy / total_energy
metadata = {"format": "pt", "fl_extraction": json.dumps({
"operation": "finetuned_model - base_model", "rank": rank,
"retained_weight_energy": retained,
})}
comfy.utils.save_torch_file(output, path, metadata=metadata)
logging.info("FL LoRA saved: %s (%.2f%% weight-difference energy retained)", path, 100 * retained)
return (path,)
+364
View File
@@ -0,0 +1,364 @@
import json
import math
import os
import uuid
from fractions import Fraction
import av
import cv2
import numpy as np
import torch
import folder_paths
import comfy.model_management as model_management
from comfy_execution.utils import get_executing_context
from server import PromptServer
from comfy_api.latest import io
from .FL_StreetScan import FL_StreetScanComposite
from .FL_VoxelNormalRelief import FL_VoxelNormalRelief
from .FL_ScanAudioEdit import FL_ScanAudioEdit, colorize_depth
from .scan_modulation import TARGETS, compile_mappings
from ..audio.audio_envelope import FLAudioEnvelope, load_audio_envelope
from ..audio.FL_Audio_Reactive_Brightness import FL_Audio_Reactive_Brightness
from ..audio.FL_Audio_Reactive_Saturation import FL_Audio_Reactive_Saturation
from ..audio.FL_Audio_Reactive_Edge_Glow import FL_Audio_Reactive_Edge_Glow
ScanAnalysis = io.Custom("FL_SCAN_ANALYSIS")
def scan_progress(stage, value=0, total=1, frame_updates=True):
context = get_executing_context()
if context is not None:
PromptServer.instance.send_sync("fl_scan_progress", {"node": context.node_id,
"stage": stage, "value": value, "max": total, "frame_updates": frame_updates})
DEFAULTS = {"orbit_degrees": 5.5, "depth_relief": 1.0, "scene_scale": .74,
"stack_count": 4, "stack_spacing": 1.0, "stack_x": 1.0, "stack_y": 1.0,
"stack_rotation": 0.0, "stack_opacity": 1.0, "stack_palette": "cobalt",
"window_order": "newest_on_top", "window_blend": "normal",
"voxel_opacity": 1.0, "edge_opacity": 1.0, "depth_opacity": 1.0,
"window_fade_in": 0.0, "window_fade_out": 0.0,
"normal_mix": 0.0, "hud_opacity": .7, "pose_opacity": .5,
"surface_seed": 41, "min_cut_frames": 10, "max_cut_frames": 20,
"cursor_scale": 1.2, "reveal_strength": 1.0,
"base_brightness": 1.0, "brightness_intensity": .16,
"base_saturation": .9, "saturation_intensity": .3,
"edge_threshold": .15, "glow_intensity": 0.0, "envelope_intensity": .28,
"glow_color": "white", "blend_mode": "screen",
"motion_mode": "current", "parallax_scope": "whole_scene", "depth_style": "grayscale",
"parallax_strength": 1.0, "offset_x": 0.0, "offset_y": 0.0, "dolly": 0.0, "steady_depth": .5,
"voxel_weight": 50.0, "edge_weight": 50.0, "depth_weight": 0.0,
"cursor_activity": 1.0, "reveal_size": 1.0, "audio_mappings": []}
LIMITS = {"orbit_degrees": (0,25), "depth_relief": (0,3), "scene_scale": (.5,1),
"stack_count": (0,8), "window_fade_in": (0,1), "window_fade_out": (0,1),
"normal_mix": (0,1), "hud_opacity": (0,1), "pose_opacity": (0,1),
"surface_seed": (0,2147481947), "min_cut_frames": (2,120), "max_cut_frames": (2,240),
"cursor_scale": (.5,3), "reveal_strength": (0,1), "base_brightness": (0,3),
"brightness_intensity": (-1,1), "base_saturation": (0,3), "saturation_intensity": (-1,1),
"edge_threshold": (0,.99), "glow_intensity": (0,2), "envelope_intensity": (0,2)}
LIMITS.update({key:bounds for key,bounds in TARGETS.items() if key in DEFAULTS})
CHOICES = {"glow_color": ["white","original","cyan","magenta","yellow"],
"stack_palette": ["cobalt","cyan","magenta","mono"],
"window_order": ["newest_on_top","voxel_on_top","edge_on_top","depth_on_top","random_on_snare"],
"window_blend": ["normal","screen","add"],
"blend_mode": ["screen","add","overlay"], "motion_mode": ["current","depth_parallax"],
"parallax_scope": ["whole_scene","reveals_only"], "depth_style": ["grayscale","false_color","contours"]}
CONTROL_HELP = {
"stack_count": "Number of back plates (0 disables them). These are 2D layers behind the projected scene, not new 3D geometry.",
"stack_spacing": "Multiplier for the original depth-relative panel spacing. Map Kick here for expanding stacks; 0 collapses the plates.",
"stack_x": "Horizontal spread direction and multiplier. Negative values spread left; 0 removes horizontal separation.",
"stack_y": "Vertical spread direction and multiplier, including the original subtle sway. Negative values spread upward.",
"stack_rotation": "Rotation in degrees per plate around the image center. Map an envelope here to fan the stack.",
"stack_opacity": "Opacity of the back plates and their outlines. Does not fade the foreground video.",
"stack_palette": "Alternating back-plate colors. Cobalt preserves the original appearance.",
"window_order": "Back-to-front reveal order. Random on snare reshuffles only on Envelope 2 rising above 0.5, using the node seed; order holds between hits.",
"window_blend": "Blend the revealed pixels with the scene below. Independent of the Finish tab's glow blend mode.",
"voxel_opacity": "Opacity multiplier for voxel-normal windows, after Reveal strength and audio accents.",
"edge_opacity": "Opacity multiplier for digital-edge windows, after Reveal strength and audio accents.",
"depth_opacity": "Opacity multiplier for depth-map windows, after Reveal strength and audio accents.",
"window_fade_in": "Reveal fade-in duration in seconds, starting when the drag begins. Borders and cursors remain visible.",
"window_fade_out": "Reveal fade-out duration in seconds before the gesture ends. Zero preserves the original abrupt end.",
"relief": "Voxel extrusion height. Start near 0.65; large values overlap neighboring cubes.",
"animation": "Amplitude of the animated voxel height variation.",
"speed": "Voxel animation speed. Zero holds the animation phase.",
"cube_size": "Voxel cell size in pixels. Larger cells produce fewer, chunkier cubes.",
"parallax_strength": "Depth-camera exaggeration. Try 0.5–1.5; large values expose missing surfaces.",
"depth_relief": "Separation between near and far depth planes. Larger values exaggerate depth.",
"dolly": "Forward/backward camera displacement. Positive values move toward the scene.",
"steady_depth": "Depth plane held steady by the camera. 0 is far, 1 is near.",
"offset_x": "Horizontal depth-camera displacement; foreground and background shift differently.",
"offset_y": "Vertical depth-camera displacement; foreground and background shift differently.",
"motion_mode": "Current preserves the original projection; depth parallax enables the native depth camera.",
"parallax_scope": "Apply the native camera to the whole scene or only the revealed layers.",
"scene_scale": "Projected scene size. Smaller values leave room for the digital frame layers.",
"orbit_degrees": "Maximum orbit angle in degrees. Large angles reveal reprojection gaps.",
"cursor_activity": "Fraction of candidate gestures retained. Zero disables gestures.",
"reveal_size": "Scale of the drag rectangle, latched when each gesture starts.",
"voxel_weight": "Relative chance of choosing voxel normals for a gesture. Zero disables this layer.",
"edge_weight": "Relative chance of choosing digital edges for a gesture. Zero disables this layer.",
"depth_weight": "Relative chance of choosing depth for a gesture. At least one layer weight must be positive.",
"normal_mix": "Blend projected normal colors into the scene.",
"hud_opacity": "Opacity of detection boxes and labels.",
"pose_opacity": "Opacity of detected pose lines.",
"motion_strength": "Strength of audio-triggered screen camera accents.",
"min_cut_frames": "Minimum audio-edit interval in frames. Raising it past the maximum also raises the maximum.",
"max_cut_frames": "Maximum audio-edit interval in frames. Lowering it below the minimum also lowers the minimum.",
"edge_threshold": "Edge sensitivity for glow. Higher values retain fewer edges.",
}
CONTROL_METADATA = {key: {"default": value, "range": LIMITS.get(key), "choices": CHOICES.get(key),
"help": CONTROL_HELP.get(key, key.replace("_", " ").capitalize() + ". Reset restores the factory value.")}
for key,value in DEFAULTS.items() if key != "audio_mappings"}
CONTROL_METADATA.update({key: {"help": CONTROL_HELP[key]} for key in ("cube_size","relief","animation","speed","motion_strength")})
CONTROL_METADATA.update(cursor_count={"help": "Number of concurrent cursor lanes. Zero disables cursor reveals."},
seed={"help": "Deterministic seed for cursor placement and editing. Keep fixed to compare settings."})
def settings_from_json(value):
settings = json.loads(value)
if not isinstance(settings, dict) or settings.keys() - DEFAULTS.keys():
raise ValueError("Interactive Scan FX: unknown advanced settings.")
settings = DEFAULTS | settings
for key, (low, high) in LIMITS.items():
number = settings[key]
if isinstance(number, bool) or not isinstance(number, (int, float)) or not math.isfinite(number) or not low <= number <= high:
raise ValueError(f"Interactive Scan FX: {key} must be between {low} and {high}.")
for key in ("surface_seed", "min_cut_frames", "max_cut_frames", "stack_count"):
if not isinstance(settings[key], int):
raise ValueError(f"Interactive Scan FX: {key} must be an integer.")
if settings["max_cut_frames"] < settings["min_cut_frames"]:
settings["min_cut_frames"], settings["max_cut_frames"] = settings["max_cut_frames"], settings["min_cut_frames"]
for name,choices in CHOICES.items():
if settings[name] not in choices:
raise ValueError(f"Interactive Scan FX: invalid {name}.")
return settings
class FL_ScanAnalysis:
@classmethod
def INPUT_TYPES(cls):
return {"required": {"depth": ("IMAGE",), "normals": ("IMAGE",)}, "optional": {
"subject_masks": ("MASK",), "detections": ("FL_SCAN_TRACKS",), "pose_keypoints": ("POSE_KEYPOINT",)}}
RETURN_TYPES = ("FL_SCAN_ANALYSIS",)
RETURN_NAMES = ("analysis",)
FUNCTION = "pack"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "Packages one shot's aligned analysis. Connect shots in chronological order to Interactive Scan FX. Does not run or load models."
def pack(self, depth, normals, subject_masks=None, detections=None, pose_keypoints=None):
if depth.ndim != 4 or normals.ndim != 4 or depth.shape[:3] != normals.shape[:3] or normals.shape[-1] != 3 or len(depth) == 0:
raise ValueError("Scan Analysis needs nonempty, aligned depth and RGB normal frames.")
return ({"depth": depth, "normals": normals, "subject_masks": subject_masks,
"detections": detections, "pose_keypoints": pose_keypoints},)
class FL_ScanVideoSection:
@classmethod
def INPUT_TYPES(cls):
return {"required": {"images": ("IMAGE",), "sections": ("INT", {"default": 4, "min": 1, "max": 100}),
"section_index": ("INT", {"default": 0, "min": 0, "max": 99})}}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "split"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "Extracts one proportional processing chunk. Indices 0 through sections-1 cover every frame exactly once, including remainder frames. This does not detect scene cuts."
def split(self, images, sections, section_index):
if not 1 <= sections <= len(images) or not 0 <= section_index < sections:
raise ValueError("Scan Video Section: use at least one frame per section and an index below the section count.")
start = len(images) * section_index // sections
end = len(images) * (section_index + 1) // sections
return (images[start:end].clone(),)
class FL_ScanVideoShots:
@classmethod
def INPUT_TYPES(cls):
return {"required": {"images": ("IMAGE",)}, "optional": {"prompt_schedule": ("FL_PROMPT_SCHEDULE",)}}
RETURN_TYPES = ("IMAGE",)
OUTPUT_IS_LIST = (True,)
FUNCTION = "split"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "Runs one shared analysis branch for every authored shot. Adjacent sections in one render group remain together. Without a schedule, processes the video as one shot."
def split(self, images, prompt_schedule=None):
if prompt_schedule is None:
return ([images],)
ranges = []
cursor = 0
previous_group = None
for section in prompt_schedule["sections"]:
start, end = section["start_frame"], section["end_frame"]
if start != cursor or end <= start or end > len(images):
raise ValueError("Scan Video Shots: schedule must cover this video in order without gaps or overlaps.")
group = section.get("render_group")
if group is not None and group == previous_group:
ranges[-1][1] = end
else:
ranges.append([start, end])
previous_group = group
cursor = end
if cursor != len(images) or not ranges:
raise ValueError("Scan Video Shots: schedule length must match the selected video. Use its original schedule or disconnect the schedule for a single shot.")
return ([images[start:end].clone() for start, end in ranges],)
class FL_ScanAnalysisCollect:
@classmethod
def INPUT_TYPES(cls):
return {"required": {"analysis": ("FL_SCAN_ANALYSIS",)}}
INPUT_IS_LIST = True
RETURN_TYPES = ("FL_SCAN_ANALYSIS",)
FUNCTION = "collect"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "Collects the shared analysis branch's ordered shots into one Scan FX input."
def collect(self, analysis):
return ({"shots": analysis},)
def write_preview(final, surface, mask, report, envelopes, fps, depth=None, curves=None, mappings=None, depth_style="grayscale"):
filename = f"fl_scan_{uuid.uuid4().hex}.mp4"
path = os.path.join(folder_paths.get_temp_directory(), filename)
height, width = final.shape[1:3]
w = min(320, width)
w -= w % 2
h = max(2, round(height * w / width / 2) * 2)
with av.open(path, mode="w") as container:
stream = container.add_stream("libx264", rate=Fraction(fps))
columns = 3 if depth is not None else 2
stream.width, stream.height, stream.pix_fmt = w * columns, h * 2, "yuv420p"
stream.options = {"crf": "20", "preset": "veryfast"}
for frame in range(len(final)):
rgb = cv2.resize(final[frame].cpu().numpy(), (w,h))
surf = cv2.resize(surface[frame].cpu().numpy(), (w,h))
matte = cv2.resize(mask[frame].cpu().numpy(), (w,h))
debug = rgb.copy()
for event in report["cursor_events"]:
if event["start"] <= frame < event["end"]:
a = tuple(np.rint(np.array(event["anchor"]) * [w,h]).astype(int))
b = tuple(np.rint(np.array(event["target"]) * [w,h]).astype(int))
cv2.arrowedLine(debug, a, b, (1,.7,.1), 1, tipLength=.15)
shot = next(s["shot"] for s in report["segments"] if s["start_frame"] <= frame < s["end_frame"])
cv2.putText(debug, f"FRAME {frame} / SHOT {shot}", (6,16), cv2.FONT_HERSHEY_PLAIN, .8, (1,1,1), 1)
tiles = [rgb,surf,np.repeat(matte[:,:,None],3,2),debug]
if depth is not None:
tiles.extend([colorize_depth(cv2.resize(depth[frame].cpu().numpy(),(w,h)),depth_style),np.zeros_like(rgb)])
atlas = np.concatenate((np.concatenate(tiles[:columns],1),np.concatenate(tiles[columns:],1)),0)
video_frame = av.VideoFrame.from_ndarray((atlas.clip(0,1)*255).astype(np.uint8), format="rgb24")
for packet in stream.encode(video_frame):
container.mux(packet)
if frame % max(1, len(final)//100) == 0:
scan_progress("Encoding previews", frame+1, len(final), False)
for packet in stream.encode():
container.mux(packet)
return {"filename": filename, "subfolder": "", "type": "temp", "fps": fps, "frames": len(final),
"width": w, "height": h, "envelopes": [e["values"] for e in envelopes],
"cuts": report["cuts"], "events": len(report["cursor_events"]), "columns": columns,
"views": ["Final","Surface","Mask","Debug"] + (["Depth"] if depth is not None else []),
"segments": report["segments"], "curves": curves or {}, "mappings": mappings or []}
class FL_InteractiveScanFX(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_InteractiveScanFX", display_name="FL Interactive Scan FX", category="🏵️Fill Nodes/VFX",
is_output_node=True,
inputs=[io.Image.Input("images"),
io.Autogrow.Input("analysis", io.Autogrow.TemplatePrefix(ScanAnalysis.Input("shot"), "shot", min=1, max=100)),
FLAudioEnvelope.Input("kick_envelope"), FLAudioEnvelope.Input("snare_envelope"), FLAudioEnvelope.Input("hihat_envelope"),
io.Int.Input("fps", default=24, min=1, max=120),
io.Int.Input("cube_size", default=12, min=4, max=64),
io.Float.Input("relief", default=.65, min=0, max=2, step=.05),
io.Float.Input("animation", default=.18, min=0, max=1, step=.01),
io.Float.Input("speed", default=.7, min=0, max=5, step=.05),
io.Int.Input("cursor_count", default=3, min=0, max=3),
io.Float.Input("motion_strength", default=.6, min=0, max=2, step=.05),
io.Int.Input("seed", default=73, min=0, max=2147483647, control_after_generate=True),
io.String.Input("advanced_settings", default=json.dumps(DEFAULTS),extra_dict={"scan_mapping_targets":TARGETS,"scan_choices":CHOICES,"scan_controls":CONTROL_METADATA})],
outputs=[io.Image.Output(display_name="final"), io.Image.Output(display_name="projected_surface"),
io.Mask.Output(display_name="reveal_mask"), io.String.Output(display_name="report")])
@classmethod
def execute(cls, images, analysis, kick_envelope, snare_envelope, hihat_envelope, fps, cube_size, relief, animation, speed, cursor_count, motion_strength, seed, advanced_settings):
settings = settings_from_json(advanced_settings)
shots = []
for key in sorted(analysis, key=lambda k: int(k.removeprefix("shot"))):
value = analysis[key]
shots.extend(value["shots"] if "shots" in value else [value])
lengths = [len(s["depth"]) for s in shots]
if not shots or sum(lengths) != len(images) or images.ndim != 4 or images.shape[-1] != 3:
raise ValueError(f"Interactive Scan FX: video has {len(images)} frames, but analysis covers {sum(lengths)} ({lengths}). Connect every ordered section from the same video; do not pad or repeat analysis frames.")
envelopes = [load_audio_envelope(e) for e in (kick_envelope,snare_envelope,hihat_envelope)]
if any(e["total_frames"] != len(images) or not math.isclose(e["fps"], fps) for e in envelopes):
raise ValueError("Interactive Scan FX: all envelopes must match video frame count and FPS.")
curves,mappings = compile_mappings(settings | {"relief":relief,"animation":animation,"speed":speed,"motion_strength":motion_strength},envelopes,fps,lengths)
scan = torch.empty_like(images, device="cpu", dtype=torch.float32)
surface = torch.empty_like(scan)
projected_depth = torch.empty(images.shape[:3],device="cpu",dtype=torch.float32)
offset = 0
for index, (shot, count) in enumerate(zip(shots,lengths)):
source = images[offset:offset+count]
if shot["depth"].shape[:3] != source.shape[:3] or shot["normals"].shape[:3] != source.shape[:3]:
raise ValueError(f"Interactive Scan FX: shot {index+1} analysis resolution does not match the video.")
shot_seed = settings["surface_seed"] + index * 17
local_curves = {key:values[offset:offset+count] for key,values in curves.items()}
animated_voxels = any(row["target"] in ("relief","animation","speed") for row in mappings)
scan_progress(f"Shot {index+1}/{len(shots)} · Voxel normals", 0, count)
voxels, = FL_VoxelNormalRelief().render_animated(shot["normals"],shot["depth"],cube_size,relief,animation,speed,fps,shot_seed,
local_curves if animated_voxels else None)
masks = shot["subject_masks"]
if masks is None:
masks = torch.zeros(source.shape[:3], dtype=torch.float32)
tracks = shot["detections"]
if tracks is None:
tracks = {"frames": [[] for _ in range(count)], "height": source.shape[1], "width": source.shape[2]}
scan_progress(f"Shot {index+1}/{len(shots)} · Depth projection", 0, count)
composite, _, projected,depth_frames = FL_StreetScanComposite().render_layers(source,shot["depth"],voxels,masks,tracks,
fps,shot_seed,settings["orbit_degrees"],settings["depth_relief"],settings["scene_scale"],0,
settings["normal_mix"],settings["hud_opacity"],0,shot["pose_keypoints"],settings["pose_opacity"],"digital_layers",
local_curves,settings["motion_mode"],settings["parallax_scope"],
settings["stack_count"],settings["stack_palette"])
scan[offset:offset+count] = composite
surface[offset:offset+count] = projected
projected_depth[offset:offset+count] = depth_frames
offset += count
del voxels, composite, projected,depth_frames
scan_progress("Cursor reveals and audio edit", 0, len(images))
final, original, mask, report_json = FL_ScanAudioEdit().render_mapped(images,scan,surface,*envelopes,
",".join(map(str,lengths)),fps,seed,settings["min_cut_frames"],settings["max_cut_frames"],1.7,
cursor_count,settings["cursor_scale"],settings["reveal_strength"],motion_strength,"audio_locked",
curves,projected_depth,settings["depth_style"],settings["window_order"],settings["window_blend"],
settings["window_fade_in"],settings["window_fade_out"])
del original, scan
quiet = {**envelopes[0],"values":[0],"total_frames":1,"duration":1/fps}
finish_device = model_management.get_torch_device()
frame_bytes = final.shape[1] * final.shape[2] * final.element_size()
if finish_device.type != "cpu" and model_management.get_free_memory(finish_device) < frame_bytes * 64 + 512 * 1024**2:
finish_device = torch.device("cpu")
scan_progress("Color and glow", 0, len(final), False)
for start in range(len(final)):
end = start+1
frames, = FL_Audio_Reactive_Brightness().apply_brightness(final[start:end],quiet,
mask=mask[start:end,:,:,None].expand(-1,-1,-1,3),base_brightness=curves["brightness"][start],brightness_intensity=0)
frames = frames.to(finish_device)
frames, = FL_Audio_Reactive_Saturation().apply_saturation(frames,quiet,base_saturation=curves["saturation"][start],saturation_intensity=0)
frames, = FL_Audio_Reactive_Edge_Glow().apply_edge_glow(frames,quiet,
glow_intensity=curves["glow"][start],envelope_intensity=0,
**{k:settings[k] for k in ("edge_threshold","glow_color","blend_mode")})
final[start:end] = frames.to(final.device)
if start % max(1, len(final)//100) == 0:
scan_progress("Color and glow", end, len(final), False)
report = json.loads(report_json)
scan_progress("Encoding previews", 0, len(final), False)
preview = write_preview(final,surface,mask,report,envelopes,fps,projected_depth,curves,mappings,settings["depth_style"])
if mappings or settings["motion_mode"] != "current" or settings["depth_weight"] > 0:
report.update(motion_mode=settings["motion_mode"],parallax_scope=settings["parallax_scope"],mappings=mappings)
report_json = json.dumps(report,indent=2)
scan_progress("Complete", 1, 1, False)
return io.NodeOutput(final,surface,mask,report_json,ui={"fl_interactive_scan":[preview]})
+298
View File
@@ -0,0 +1,298 @@
import math
import torch
import torch.nn.functional as F
import comfy.model_management as mm
import nodes
from comfy.utils import ProgressBar
from comfy_api.latest import io
Layer = io.Custom("FL_PARALLAX_LAYER")
class FL_ParallaxStackFromBatch(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_ParallaxStackFromBatch", display_name="FL Parallax Stack From RGBA Batch", category="Fill Nodes/VFX", inputs=[
io.Image.Input("images", tooltip="RGBA layers ordered back to front, background first. Exclude any reconstruction/composite image."),
io.Float.Input("near_depth", default=2, min=.25, max=100, step=.25),
io.Float.Input("far_depth", default=10, min=.25, max=100, step=.25),
], outputs=[io.Custom("FL_PARALLAX_STACK").Output(display_name="layer_stack")])
@classmethod
def execute(cls, images, near_depth, far_depth):
if images.ndim != 4 or not len(images) or images.shape[-1] != 4:
raise ValueError("Parallax stack needs a nonempty RGBA layer batch, background first.")
if not all(math.isfinite(d) and d > 0 for d in (near_depth, far_depth)) or near_depth > far_depth:
raise ValueError("Parallax stack depths must be positive, with near depth <= far depth.")
count = len(images) - 1
layers = []
for i in range(count):
depth = far_depth - (far_depth - near_depth) * i / max(1, count - 1) if count > 1 else near_depth
layers.append(dict(images=images[i + 1:i + 2].clone(), mask=None, depth=depth,
scale=1, offset_x=0, offset_y=0, opacity=1, name=f"Layer {i + 1}", kind="art"))
return io.NodeOutput(dict(background=images[:1].clone(), layers=layers))
class FL_ParallaxDepthSources(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_ParallaxDepthSources", display_name="FL Parallax Depth Sources", category="Fill Nodes/VFX", inputs=[
io.Custom("FL_PARALLAX_STACK").Input("layer_stack"),
io.Int.Input("resolution", default=518, min=126, max=1024, step=14),
], outputs=[io.Image.Output(display_name="depth_analysis_images")])
@classmethod
def execute(cls, layer_stack, resolution):
images = [layer_stack["background"]] + [p["images"] for p in layer_stack["layers"]]
h, w = images[0].shape[1:3]
size = (max(14, round(h * resolution / max(h, w) / 14) * 14), max(14, round(w * resolution / max(h, w) / 14) * 14))
sources = []
for image in images:
if len(image) != 1:
raise ValueError("Parallax depth analysis currently supports still-image layers only.")
rgb = image[..., :3]
if image.shape[-1] == 4:
alpha = image[..., 3:4].clamp(0, 1)
rgb = rgb * alpha + .5 * (1 - alpha)
sources.append(F.interpolate(rgb.movedim(-1, 1), size=size, mode="bilinear", align_corners=False).movedim(1, -1))
return io.NodeOutput(torch.cat(sources))
def prepare_relief(depth_map, invert, smoothing, device):
if depth_map.ndim != 4 or depth_map.shape[-1] < 1 or not torch.isfinite(depth_map).all():
raise ValueError("Parallax relief needs a finite IMAGE depth batch, background first then cutouts in stack order.")
depth = depth_map[..., :3].mean(-1, keepdim=True).movedim(-1, 1).to(device=device, dtype=torch.float32).clamp(0, 1)
if invert:
depth = 1 - depth
radius = int(smoothing)
if radius:
depth = F.avg_pool2d(F.pad(depth, (radius,) * 4, mode="replicate"), 2 * radius + 1, stride=1)
return depth
class FL_ParallaxLayer(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_ParallaxLayer", display_name="FL Parallax Layer", category="Fill Nodes/VFX", inputs=[
io.Image.Input("images", tooltip="RGBA from a matting node, or RGB with an explicit foreground mask."),
io.Float.Input("depth", default=4, min=0.25, max=100, step=0.25, tooltip="Camera distance. Smaller values move faster and draw in front."),
io.Float.Input("scale", default=1, min=0.1, max=4, step=0.01),
io.Float.Input("offset_x", default=0, min=-2, max=2, step=0.01, tooltip="Fraction of output width. Positive moves right."),
io.Float.Input("offset_y", default=0, min=-2, max=2, step=0.01, tooltip="Fraction of output height. Positive moves down."),
io.Float.Input("opacity", default=1, min=0, max=1, step=0.01),
io.Mask.Input("mask", optional=True, tooltip="White = opaque foreground. Overrides embedded alpha."),
], outputs=[Layer.Output(display_name="layer"), io.Image.Output(display_name="matte_check")])
@classmethod
def execute(cls, images, depth, scale, offset_x, offset_y, opacity, mask=None):
if images.ndim != 4 or images.shape[-1] not in (3, 4) or len(images) == 0:
raise ValueError("Parallax Layer needs a nonempty RGB or RGBA image batch.")
if not all(math.isfinite(v) for v in (depth, scale, offset_x, offset_y, opacity)) or depth <= 0 or scale <= 0 or not 0 <= opacity <= 1:
raise ValueError("Parallax Layer needs positive depth/scale and opacity between zero and one.")
if mask is not None and (mask.ndim != 3 or mask.shape[1:3] != images.shape[1:3] or len(mask) not in (1, len(images))):
raise ValueError("Parallax mask must match image size, with one frame or one mask per image.")
if mask is None and images.shape[-1] != 4:
raise ValueError("Connect RGBA images or a foreground MASK. RGB alone has no cutout alpha.")
layer = dict(images=images, mask=mask, depth=depth, scale=scale, offset_x=offset_x, offset_y=offset_y, opacity=opacity)
i = len(images) // 2
rgb = images[i:i+1, ..., :3]
alpha = (mask[0:1] if len(mask) == 1 else mask[i:i+1]) if mask is not None else images[i:i+1, ..., 3]
h, w = images.shape[1:3]
y = torch.arange(h, device=rgb.device)[:, None] // 16
x = torch.arange(w, device=rgb.device)[None, :] // 16
checker = (0.22 + ((x + y) % 2) * 0.12).to(rgb.dtype)[None, ..., None]
alpha = alpha.to(rgb.device)[..., None].clamp(0, 1) * opacity
preview = rgb * alpha + checker * (1 - alpha)
return io.NodeOutput(layer, preview)
def camera_path(count, motion):
if count == 1 or motion == "locked":
return [0.0] * count
values = []
for i in range(count):
t = i / (count - 1)
if motion == "loop":
value = math.sin(t * math.tau)
elif motion == "glide":
value = 2 * (t * t * (3 - 2 * t)) - 1
else:
keys = ((0, -1), (.10, -1), (.28, .4), (.45, .4), (.61, -.3), (.73, -.3), (.92, 1), (1, 1))
for (ta, va), (tb, vb) in zip(keys, keys[1:]):
if ta <= t <= tb:
u = (t - ta) / (tb - ta)
value = va + (vb - va) * (u * u * (3 - 2 * u))
break
values.append(value)
return values
def project_plate(images, mask, start, stop, grid, depth, scale, offset_x, offset_y, opacity, xs, ys, push, width, height, opaque=False, fit="cover", relief_map=None, relief_strength=0, relief_anchor=.5):
batch = images[0:1] if len(images) == 1 else images[start:stop]
batch = batch.to(device=grid.device, dtype=torch.float32)
rgb = batch[..., :3].movedim(-1, 1)
if opaque:
alpha = torch.ones_like(rgb[:, :1])
elif mask is not None:
alpha = mask[0:1].expand(len(batch), -1, -1) if len(mask) == 1 else mask[start:stop]
alpha = alpha.to(device=grid.device, dtype=torch.float32).unsqueeze(1).clamp(0, 1)
else:
alpha = batch[..., 3:4].movedim(-1, 1).clamp(0, 1)
alpha = alpha * opacity
source = torch.cat((rgb * alpha, alpha), dim=1)
if len(images) == 1:
source = source.expand(stop-start, -1, -1, -1)
ih, iw = images.shape[1:3]
cover = (min if fit == "contain" else max)(width / iw, height / ih)
sx, sy = iw * cover / width * scale, ih * cover / height * scale
zoom = depth / (depth - push)
# Inverse pinhole projection of fronto-parallel planes; neutral framing is independent of depth.
gx = (grid[..., 0] / zoom[:, None, None] + 2 * xs[:, None, None] / depth - 2 * offset_x) / sx
gy = (grid[..., 1] / zoom[:, None, None] + 2 * ys[:, None, None] / depth - 2 * offset_y) / sy
sample_grid = torch.stack((gx, gy), -1)
if relief_map is not None and relief_strength:
depth_source = relief_map.expand(stop-start, -1, -1, -1)
# Two inverse-warp refinements keep RGB and premultiplied alpha on the same surface.
for _ in range(2):
sampled = F.grid_sample(depth_source, sample_grid, padding_mode="border", align_corners=False)[:, 0]
inv_depth = (1 + relief_strength * (sampled - relief_anchor)) / depth
gx = (grid[..., 0] * (1 - push[:, None, None] * inv_depth) + 2 * xs[:, None, None] * inv_depth - 2 * offset_x) / sx
gy = (grid[..., 1] * (1 - push[:, None, None] * inv_depth) + 2 * ys[:, None, None] * inv_depth - 2 * offset_y) / sy
sample_grid = torch.stack((gx, gy), -1)
return F.grid_sample(source, sample_grid, mode="bilinear", padding_mode="border" if opaque else "zeros", align_corners=False)
def preview_layers(background, plates):
previews = []
for i, p in enumerate([dict(images=background, mask=None)] + plates):
image = p["images"][:1].detach().cpu()
rgb = image[..., :3]
alpha = p["mask"][:1].detach().cpu()[..., None] if p["mask"] is not None else image[..., 3:4]
if i == 0 or image.shape[-1] == 3 and p["mask"] is None:
alpha = torch.ones_like(rgb[..., :1])
rgba = torch.cat((rgb, alpha.clamp(0, 1)), -1)
h, w = rgba.shape[1:3]
rgba = F.interpolate(rgba.movedim(-1, 1), size=(max(1, round(h * 256 / max(h, w))), max(1, round(w * 256 / max(h, w)))), mode="area").movedim(1, -1)
file = nodes.SaveImage().save_images(rgba, "Parallax/previews/plate")["ui"]["images"][0]
previews.append(dict(image=file, width=w, height=h, background=i == 0, name=p.get("name", "Background" if i == 0 else f"Layer {i}"),
animated=len(p["images"]) > 1, kind=p.get("kind", "art"), **{k: p.get(k, v) for k, v in dict(depth=12, scale=1, offset_x=0, offset_y=0, opacity=1).items()}))
return previews
class FL_LayeredParallax(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_LayeredParallax", display_name="FL Layered Parallax", category="Fill Nodes/VFX", inputs=[
io.Image.Input("background", optional=True, tooltip="Opaque distant plate. Optional when a poster layer stack supplies the background."),
io.Autogrow.Input("layers", optional=True, template=io.Autogrow.TemplatePrefix(input=Layer.Input("layer"), prefix="layer_", min=0, max=100)),
io.Int.Input("width", default=640, min=64, max=4096, step=8),
io.Int.Input("height", default=360, min=64, max=4096, step=8),
io.Combo.Input("motion", options=["bursts", "glide", "loop", "locked"], default="bursts"),
io.Float.Input("travel_x", default=0.35, min=-2, max=2, step=0.01, tooltip="Camera excursion in output widths at depth 1. Actual displacement divides by layer depth."),
io.Float.Input("travel_y", default=0.035, min=-2, max=2, step=0.005),
io.Float.Input("push_in", default=0.06, min=-0.2, max=0.2, step=0.01, tooltip="Forward camera travel in depth units; zero disables dolly scaling."),
io.Float.Input("background_depth", default=12, min=0.25, max=100, step=0.25),
io.Float.Input("overscan", default=1.12, min=1, max=2, step=0.01, tooltip="Extra background coverage. Border extension is used beyond the source; no generated hidden scenery."),
io.Combo.Input("device", options=["auto", "cpu"], default="auto"),
io.Int.Input("frames", optional=True, default=0, min=0, max=4096, tooltip="Output frames for still-image plates. Zero uses the source video length. Animated plates must match this length."),
io.Custom("FL_PARALLAX_STACK").Input("layer_stack", optional=True, tooltip="Dynamic poster layers, including their background. Stack background takes precedence over the separate image input."),
io.Combo.Input("layer_fit", optional=True, options=["cover", "contain"], default="cover", tooltip="Cover fills the output aspect ratio; contain preserves the whole cutout when source and output shapes differ. Background always covers the canvas."),
io.Image.Input("depth_maps", optional=True, tooltip="One background map, or a depth batch from FL Parallax Depth Sources in unchanged stack order. White means near."),
io.Combo.Input("relief_scope", optional=True, options=["off", "background", "background + artwork"], default="off"),
io.Float.Input("relief_strength", optional=True, default=.2, min=0, max=.5, step=.01),
io.Float.Input("relief_anchor", optional=True, default=.5, min=0, max=1, step=.01),
io.Boolean.Input("depth_invert", optional=True, default=False),
io.Int.Input("depth_smoothing", optional=True, default=2, min=0, max=16),
], outputs=[io.Image.Output(display_name="parallax"), io.Image.Output(display_name="locked_composite"), io.Image.Output(display_name="depth_preview")])
@classmethod
def execute(cls, background=None, width=640, height=360, motion="bursts", travel_x=.35, travel_y=.035, push_in=.06, background_depth=12, overscan=1.12, device="auto", layers=None, frames=0, layer_stack=None, layer_fit="cover", depth_maps=None, relief_scope="off", relief_strength=.2, relief_anchor=.5, depth_invert=False, depth_smoothing=2):
if layer_stack is not None:
background = layer_stack["background"]
if background is None or background.ndim != 4 or background.shape[-1] < 3 or len(background) == 0:
raise ValueError("Layered Parallax needs a nonempty background image batch.")
plates = [v for v in (layers or {}).values() if v is not None]
if layer_stack is not None:
plates.extend(layer_stack["layers"])
plates = [dict(p) for p in plates]
for i, p in enumerate(plates):
p["depth_index"] = i + 1
active_relief = relief_scope != "off" and relief_strength > 0
render_device = mm.get_torch_device() if device == "auto" else torch.device("cpu")
background_relief = None
if active_relief:
if not 0 <= relief_strength <= .5 or not 0 <= relief_anchor <= 1 or not 0 <= depth_smoothing <= 16:
raise ValueError("Parallax relief: strength must be 0–0.5, anchor 0–1, smoothing 0–16.")
if depth_maps is None or len(depth_maps) not in (1, len(plates) + 1):
raise ValueError("Parallax relief needs one background depth map or one map for every plate, background first.")
if len(background) != 1 or any(len(p["images"]) != 1 for p in plates):
raise ValueError("Parallax relief currently supports still-image layers only; use Off for animated plates.")
if relief_scope == "background + artwork" and plates and len(depth_maps) == 1:
raise ValueError("Artwork relief needs the complete depth batch from FL Parallax Depth Sources.")
prepared = prepare_relief(depth_maps, depth_invert, depth_smoothing, render_device)
background_relief = prepared[:1]
for i, p in enumerate(plates):
if relief_scope == "background + artwork" and p.get("kind") != "text":
p["relief_map"] = prepared[i+1:i+2]
plates.sort(key=lambda p: p["depth"], reverse=True)
count = frames or max([len(background)] + [len(p["images"]) for p in plates])
if len(background) not in (1, count) or any(len(p["images"]) not in (1, count) for p in plates):
raise ValueError("Parallax videos must have equal frame counts; single images can be held for the full clip. Trim or resample mismatched videos first.")
if not all(math.isfinite(v) for v in (travel_x, travel_y, push_in, background_depth, overscan)) or background_depth <= abs(push_in) or any(p["depth"] <= abs(push_in) for p in plates):
raise ValueError("Camera push must stay in front of every layer; use positive depths greater than the push distance.")
if any(p["depth"] >= background_depth for p in plates):
raise ValueError("Background depth must be greater than each cutout layer depth.")
if active_relief and any(p["depth"] / (1 + relief_strength) <= abs(push_in) for p in [dict(depth=background_depth)] + plates):
raise ValueError("Reduce camera push or relief: the camera must stay in front of the relieved surface.")
x = (torch.arange(width, device=render_device, dtype=torch.float32) + .5) * (2 / width) - 1
y = (torch.arange(height, device=render_device, dtype=torch.float32) + .5) * (2 / height) - 1
yy, xx = torch.meshgrid(y, x, indexing="ij")
grid = torch.stack((xx, yy), -1).unsqueeze(0)
path = camera_path(count, motion)
result = torch.empty((count, height, width, 3), dtype=torch.float32)
static = len(background) == 1 and all(len(p["images"]) == 1 for p in plates)
locked = torch.empty((1 if static else count, height, width, 3), dtype=torch.float32)
depth_preview = None
progress = ProgressBar(count)
for start in range(0, count, 4):
mm.throw_exception_if_processing_interrupted()
stop = min(count, start + 4)
t = torch.tensor(path[start:stop], device=render_device, dtype=torch.float32)
xs, ys, push = t * travel_x, t * travel_y, t * push_in
neutral = not static or start == 0
flat_start, flat_stop = (0, 1) if static else (start, stop)
zero = torch.zeros(flat_stop-flat_start, device=render_device, dtype=torch.float32)
out = project_plate(background, None, start, stop, grid, background_depth, overscan, 0, 0, 1, xs, ys, push, width, height, True, relief_map=background_relief, relief_strength=relief_strength, relief_anchor=relief_anchor)[:, :3]
if neutral:
flat = project_plate(background, None, flat_start, flat_stop, grid, background_depth, overscan, 0, 0, 1, zero, zero, zero, width, height, True)[:, :3]
middle = count // 2 - start
depth = torch.zeros_like(out[:1]) if 0 <= middle < stop-start else None
for p in plates:
args = (p["images"], p["mask"], start, stop, grid, p["depth"], p["scale"], p["offset_x"], p["offset_y"], p["opacity"])
rgba = project_plate(*args, xs, ys, push, width, height, fit=layer_fit, relief_map=p.get("relief_map"), relief_strength=relief_strength, relief_anchor=relief_anchor)
out = out * (1 - rgba[:, 3:4]) + rgba[:, :3]
if neutral:
still = project_plate(p["images"], p["mask"], flat_start, flat_stop, grid, p["depth"], p["scale"], p["offset_x"], p["offset_y"], p["opacity"], zero, zero, zero, width, height, fit=layer_fit)
flat = flat * (1 - still[:, 3:4]) + still[:, :3]
if depth is not None:
alpha = rgba[middle:middle+1, 3:4]
depth = depth * (1 - alpha) + (1 - p["depth"] / background_depth) * alpha
result[start:stop] = out.movedim(1, -1).clamp(0, 1).cpu()
if neutral:
locked[flat_start:flat_stop] = flat.movedim(1, -1).clamp(0, 1).cpu()
if depth is not None:
depth_preview = depth.movedim(1, -1).cpu()
progress.update(stop-start)
if static:
locked = locked.expand(count, -1, -1, -1)
previews = preview_layers(background, plates)
for preview, index in zip(previews, [0] + [p["depth_index"] for p in plates]):
if depth_maps is not None and index < len(depth_maps):
raw = prepare_relief(depth_maps[index:index+1], False, depth_smoothing, torch.device("cpu"))
small = F.interpolate(raw, size=(32, 32), mode="bilinear", align_corners=False)[0, 0].cpu().clamp(0, 1)
preview["relief"] = small.flatten().tolist()
return io.NodeOutput(result, locked, depth_preview, ui={"parallax_layers": previews,
"parallax_settings": [dict(width=width, height=height, motion=motion, travel_x=travel_x, travel_y=travel_y, push_in=push_in, background_depth=background_depth, overscan=overscan, device=device, layer_fit=layer_fit, relief_scope=relief_scope, relief_strength=relief_strength, relief_anchor=relief_anchor, depth_invert=depth_invert, depth_smoothing=depth_smoothing)]})
+232
View File
@@ -0,0 +1,232 @@
import hashlib
import json
import math
import re
import torch.nn.functional as F
import comfy.samplers
import nodes
from comfy_api.latest import io
from comfy_execution.graph_utils import GraphBuilder
from comfy_extras.nodes_textgen import TextGenerate
Plan = io.Custom("FL_POSTER_LAYER_PLAN")
Asset = io.Custom("FL_POSTER_LAYER_ASSET")
Stack = io.Custom("FL_PARALLAX_STACK")
def parse_plan(text, maximum):
text = re.sub(r"<think>.*?</think>", "", text, flags=re.S).strip()
if text.startswith("```"):
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text)
try:
entries = json.loads(text)
except json.JSONDecodeError as error:
raise ValueError("Poster planner: expected a JSON layer list. Review the generated text or use Manual mode.") from error
if not isinstance(entries, list):
raise ValueError("Image layer planner: return a JSON array, not an enclosing object.")
if not 1 <= len(entries) <= maximum:
raise ValueError(f"Image layer planner returned {len(entries)} layers; use between 1 and {maximum}, including the background.")
result = []
for i, entry in enumerate(entries):
if not isinstance(entry, dict) or entry.get("kind") not in ("background", "art", "text"):
raise ValueError("Poster planner: each layer needs kind background, art or text.")
if (entry["kind"] == "background") != (i == 0):
raise ValueError("Poster planner: the first layer must be the only background.")
name, prompt = entry.get("name"), entry.get("prompt")
if not isinstance(name, str) or not name.strip() or not isinstance(prompt, str) or not prompt.strip():
raise ValueError("Poster planner: each layer needs a name and a basic-English extraction prompt.")
depth = entry.get("depth", 12 if i == 0 else max(1, 10 - i))
if not isinstance(depth, (int, float)) or not math.isfinite(depth) or not .25 <= depth <= 12 or (i and depth > 11.75):
raise ValueError("Poster planner: cutout depths must be between 0.25 and 11.75; background depth is 12.")
result.append(dict(id=f"layer_{i}", name=name.strip(), prompt=prompt.strip(), kind=entry["kind"], depth=12 if i == 0 else depth))
return result
def plan_key(plan):
return hashlib.sha256(json.dumps(plan, sort_keys=True).encode()).hexdigest()[:20]
def apply_overrides(plan, text):
data = json.loads(text)
if not isinstance(data, dict):
raise ValueError("Poster layers: overrides must be a JSON object.")
overrides = data.get("layers", {}) if data.get("plan_key") == plan_key(plan) else {}
if not isinstance(overrides, dict):
raise ValueError("Poster layers: layer edits must be keyed by layer ID.")
result = []
for entry in plan:
item = dict(entry, visible=True, scale=1.04, offset_x=0, offset_y=0, revision=0)
item["defaults"] = dict(item)
edit = overrides.get(entry["id"], {})
if not isinstance(edit, dict):
raise ValueError("Poster layers: each override must be an object.")
for key in ("prompt", "depth", "visible", "scale", "offset_x", "offset_y", "revision"):
if key in edit:
item[key] = edit[key]
if not isinstance(item["prompt"], str) or not item["prompt"].strip():
raise ValueError("Poster layers: extraction prompts cannot be empty.")
for key, low, high in (("depth", .25, 11.75), ("scale", .1, 4), ("offset_x", -2, 2), ("offset_y", -2, 2)):
if entry["kind"] == "background" and key == "depth":
item[key] = 12
continue
value = item[key]
if not isinstance(value, (int, float)) or not math.isfinite(value) or not low <= value <= high:
raise ValueError(f"Poster layers: {key} must be between {low} and {high}.")
if not isinstance(item["visible"], bool) or not isinstance(item["revision"], int) or not 0 <= item["revision"] <= 1000000:
raise ValueError("Poster layers: visibility must be boolean and reroll revision a nonnegative integer.")
if entry["kind"] == "background":
item["visible"] = True
result.append(item)
return result
class FL_PosterLayerPlanner(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_PosterLayerPlanner", display_name="FL Image Layer Planner", category="Fill Nodes/VFX/Poster", inputs=[
io.Image.Input("image"), io.Clip.Input("vision_clip", optional=True, lazy=True),
io.Combo.Input("mode", options=["auto", "manual"], default="auto"),
io.Int.Input("maximum_layers", default=6, min=1, max=32, tooltip="Extraction budget including the background; Auto may choose fewer layers."),
io.Combo.Input("grouping", options=["broad groups", "balanced", "individual elements"], default="balanced"),
io.String.Input("manual_plan", default='[{"name":"Background","kind":"background","prompt":"Plain paper background without text or objects","depth":12}]', multiline=True),
io.Combo.Input("artwork_type", optional=True, options=["auto", "photography", "illustration", "graphic design", "product / 3D", "painting / collage"], default="auto", tooltip="Guides layer grouping, not style transfer. Auto follows the actual image; text layers are only planned when text is visible."),
], outputs=[Plan.Output(display_name="plan"), io.String.Output(display_name="plan_json")])
@classmethod
def check_lazy_status(cls, mode, vision_clip=None, **kwargs):
return ["vision_clip"] if mode == "auto" and vision_clip is None else []
@classmethod
def execute(cls, image, mode, maximum_layers, grouping, manual_plan, vision_clip=None, artwork_type="auto"):
if len(image) != 1:
raise ValueError("Poster planner: select one poster image, not a video or batch.")
if mode == "auto":
if vision_clip is None:
raise ValueError("Poster planner: connect a vision-capable CLIP for Auto mode.")
prompt = f'''Inspect this image and plan its extraction into independently moving 2.5D layers. Artwork type: {artwork_type}; auto means infer the medium from the actual image, not from a presumed poster template.
Return ONLY a valid JSON array, with at most {maximum_layers} entries INCLUDING one background. Choose fewer when appropriate. Grouping preference: {grouping}.
Each entry must have: "name" (short label), "kind" ("background", "art", or "text"), "prompt" (a short basic English noun phrase identifying the visible material to extract), "depth" (number).
The first entry must be the only background, depth 12: describe ALL permanent scenery and support surfaces remaining after the cutouts are removed, not just the farthest wall or sky. Include visible floors, ground, tabletops and room surfaces together so objects do not float in an empty backdrop. Preserve their material, perspective and lighting. Do not replace a landscape, room or photographic setting with paper. For graphic layouts, the background is the full canvas or paper ONLY: floating disks, panels and color blocks must be separate art cutouts, never mixed into the background prompt. Group related graphics or matching props into a shared cutout when the budget is tight. Other depths must be 1 through 10; larger depths are behind smaller ones.
Only if lettering is actually visible, include meaningful text groups as text layers, preserving their exact visible words. Do not invent a headline or add text to a text-free image. Describe lettering style and location. Only name a lettering color if clearly visible; do not confuse the ink color with its label background. Never rewrite words.
Name whole subjects with their worn accessories. Group matching decorations and identify all their positions, without guessing their count. In each extraction prompt, name ONLY the target objects. Use image directions for location (upper right, lower left), never other objects as location anchors: write "silver four-point stars along the left and right edges", NOT "stars around the woman and speakers". Do not assign the same object to multiple layers. Do not invent invisible objects. Keep at least a background and the principal foreground material when your budget permits.
Preserve the source medium, photographic realism, brushwork, line work, texture and lighting. For photos and products, keep attached shadows, reflections and fine hair with the relevant subject where separable extraction would break them. Keep continuous surfaces together. For landscapes, prefer coherent foreground, middle ground and distant scenery instead of individual leaves. For graphic design, group related lettering and shapes. Do not split a subject's body into pieces. Fine transparency and reflections may not survive extraction perfectly; prefer fewer coherent layers over fragile tiny fragments.
Use all layers together to reconstruct the complete image. Depth follows visible occlusion, not a fixed person-and-headline template. No Markdown, commentary, bounding boxes or coordinate arrays.'''
prompt += '\nAssign depth from visible occlusion, NOT list order: distant material 9, middle ground 6, foreground 2 are useful starting points. Worn accessories belong with their subject, not separate layers. Stay within the total layer budget by grouping related elements.'
text = TextGenerate.execute(vision_clip, prompt, max(1536, maximum_layers * 192), {"sampling_mode": "off"}, image=image, thinking=True).result[0]
try:
plan = parse_plan(text, maximum_layers)
except ValueError as error:
correction = f"{prompt}\nRevise this draft. Validation failed: {error}\nMerge related objects or lettering to respect the budget, preserving all visible material. Keep subjects and their worn accessories together. Return only the corrected JSON array.\nDraft:\n{text}"
text = TextGenerate.execute(vision_clip, correction, max(1536, maximum_layers * 192), {"sampling_mode": "off"}, image=image, thinking=True).result[0]
plan = parse_plan(text, maximum_layers)
else:
text = manual_plan
plan = parse_plan(text, maximum_layers)
if mode == "auto":
targets = {}
for entry in plan:
if entry["kind"] != "background":
# Other-object location anchors make Control extract those objects too.
entry["prompt"] = re.split(r"\s+(?:(?:positioned|located|placed)\s+)?(?:behind|beside|around|surrounding|above|below|in front of|(?:on\s+)?(?:either|both|the left|the right)\s+sides?\s+of|to the (?:left|right) of)\b", entry["prompt"], maxsplit=1, flags=re.I)[0].rstrip(" ,;")
key = (entry["kind"], " ".join(entry["prompt"].casefold().split()))
if key in targets:
targets[key]["name"] += " / " + entry["name"]
else:
targets[key] = entry
plan = [dict(entry, id=f"layer_{i}") for i, entry in enumerate(targets.values())]
rendered = json.dumps(plan, ensure_ascii=False, indent=2)
return io.NodeOutput(plan, rendered, ui={"poster_plan": [plan]})
def extraction_graph(plan, model, clip, vae, image, seed, steps, cfg, sampler_name, scheduler, overrides, display_id=None):
entries = apply_overrides(plan, overrides)
graph = GraphBuilder()
size = graph.node("GetImageSize", "size", image=image)
init = graph.node("EmptyQwenImageLayeredLatentImage", "init", width=size.out(0), height=size.out(1), layers=0, batch_size=1)
reference = graph.node("VAEEncode", "reference", pixels=image, vae=vae)
negative = graph.node("CLIPTextEncode", "negative", clip=clip, text="")
negative_ref = graph.node("ReferenceLatent", "negative_ref", conditioning=negative.out(0), latent=reference.out(0))
assets = {}
for i, entry in enumerate(entries):
key = entry["id"]
positive = graph.node("CLIPTextEncode", key + "_text", clip=clip, text=entry["prompt"])
cond = graph.node("ReferenceLatent", key + "_ref", conditioning=positive.out(0), latent=reference.out(0))
sample = graph.node("KSampler", key + "_sample", model=model, positive=cond.out(0), negative=negative_ref.out(0), latent_image=init.out(0),
seed=(seed + i + entry["revision"] * 100003) % (2**64), steps=steps, cfg=cfg, sampler_name=sampler_name, scheduler=scheduler, denoise=1)
decode = graph.node("VAEDecode", key + "_decode", samples=sample.out(0), vae=vae)
asset = graph.node("FL_PosterLayerAsset", key + "_asset", image=decode.out(0), layer_id=key)
assets["assets.asset_" + str(i)] = asset.out(0)
stack = graph.node("FL_PosterLayerStack", "stack", layout=json.dumps(entries), plan_key=plan_key(plan), **assets)
if display_id is not None:
stack.set_override_display_id(display_id)
return io.NodeOutput(stack.out(0), expand=graph.finalize())
class FL_PosterLayers(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_PosterLayers", display_name="FL Poster Layers", category="Fill Nodes/VFX/Poster", enable_expand=True, inputs=[
Plan.Input("plan"), io.Model.Input("model", raw_link=True), io.Clip.Input("clip", raw_link=True),
io.Vae.Input("vae", raw_link=True), io.Image.Input("image", raw_link=True),
io.Int.Input("seed", default=777, min=0, max=2**64-1, control_after_generate=True),
io.Int.Input("steps", default=30, min=1, max=100), io.Float.Input("cfg", default=4, min=0, max=20, step=.1),
io.Combo.Input("sampler_name", options=comfy.samplers.KSampler.SAMPLERS, default="euler"),
io.Combo.Input("scheduler", options=comfy.samplers.KSampler.SCHEDULERS, default="simple"),
io.String.Input("overrides", default="{}", multiline=True, tooltip="Per-layer prompt, depth, placement, visibility and reroll edits. Applied on the next Run."),
], outputs=[Stack.Output(display_name="layer_stack")], hidden=[io.Hidden.unique_id])
@classmethod
def execute(cls, plan, model, clip, vae, image, seed, steps, cfg, sampler_name, scheduler, overrides):
return extraction_graph(plan, model, clip, vae, image, seed, steps, cfg, sampler_name, scheduler, overrides, cls.hidden.unique_id)
class FL_PosterLayerAsset(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_PosterLayerAsset", category="Fill Nodes/VFX/Poster/Internal", is_output_node=True, inputs=[
io.Image.Input("image", lazy=True), io.String.Input("layer_id", default="layer_0"),
], outputs=[Asset.Output()])
@classmethod
def check_lazy_status(cls, image=None, **kwargs):
# Give the disk cache a lookup before scheduling extraction ancestors.
return ["image"] if image is None else []
@classmethod
def execute(cls, image, layer_id):
if not re.fullmatch(r"layer_\d+", layer_id):
raise ValueError("Poster asset: invalid layer ID.")
if image.ndim != 4 or len(image) != 1 or image.shape[-1] != 4:
raise ValueError("Poster extraction needs one RGBA image per layer. Use Qwen Layered Control and its RGBA VAE.")
saved = nodes.SaveImage().save_images(image, "Dynamic_Parallax_Poster/layers/" + layer_id)["ui"]["images"][0]
h, w = image.shape[1:3]
thumb = F.interpolate(image.movedim(-1, 1), size=(max(1, round(h * 192 / max(h, w))), max(1, round(w * 192 / max(h, w)))), mode="area").movedim(1, -1)
thumbnail = nodes.SaveImage().save_images(thumb, "Dynamic_Parallax_Poster/thumbnails/" + layer_id)["ui"]["images"][0]
coverage = float((image[..., 3] > .05).float().mean())
return io.NodeOutput(dict(image=image, file=saved, thumbnail=thumbnail, coverage=coverage),
ui={"poster_layer_ready": [dict(id=layer_id, file=saved, thumbnail=thumbnail)]})
class FL_PosterLayerStack(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id="FL_PosterLayerStack", category="Fill Nodes/VFX/Poster/Internal", inputs=[
io.String.Input("layout", default="[]"), io.String.Input("plan_key", default=""),
io.Autogrow.Input("assets", template=io.Autogrow.TemplatePrefix(input=Asset.Input("asset"), prefix="asset_", min=1, max=100)),
], outputs=[Stack.Output()])
@classmethod
def execute(cls, layout, plan_key, assets):
entries = json.loads(layout)
if len(entries) != len(assets) or not entries or entries[0]["kind"] != "background":
raise ValueError("Poster stack: the assets must match the plan, starting with its background.")
layers, review = [], []
for i, entry in enumerate(entries):
asset = assets["asset_" + str(i)]
review.append(dict(entry, file=asset["file"], thumbnail=asset["thumbnail"], coverage=asset["coverage"]))
if i and entry["visible"]:
layers.append(dict(images=asset["image"], mask=None, depth=entry["depth"], scale=entry["scale"], offset_x=entry["offset_x"], offset_y=entry["offset_y"], opacity=1, name=entry["name"], kind=entry["kind"]))
return io.NodeOutput(dict(background=assets["asset_0"]["image"], layers=layers), ui={"poster_layers": review, "poster_plan_key": [plan_key]})
+304
View File
@@ -0,0 +1,304 @@
"""Beat-driven editing and cursor-drag reveals of aligned scan layers."""
import json
import math
import random
import cv2
import numpy as np
import torch
from comfy.utils import ProgressBar
from ..audio.audio_envelope import load_audio_envelope
def edit_plan(lengths, kick, snare, min_cut, max_cut, rate, seed):
if not lengths or any(length < 1 for length in lengths):
raise ValueError("Scan Edit shot lengths must be positive frame counts.")
if min_cut < 2 or max_cut < min_cut:
raise ValueError("Scan Edit maximum cut length must be at least the minimum (2 frames or more).")
if not math.isfinite(rate) or rate <= 0:
raise ValueError("Scan Edit playback rate must be positive.")
count = len(kick)
triggers = [max(k, s) for k, s in zip(kick, snare)]
boundaries, reasons = [0], ["start"]
for frame in range(1, count):
age = frame - boundaries[-1]
hit = triggers[frame] >= 0.65 and triggers[frame - 1] < 0.65
if age >= min_cut and (hit or age >= max_cut):
boundaries.append(frame)
reasons.append("audio_onset" if hit else "maximum_hold")
boundaries.append(count)
rng = np.random.default_rng(seed)
offsets = [sum(lengths[:index]) for index in range(len(lengths))]
indices, segments, cycle = [], [], []
previous_shot = None
for index, (start, end) in enumerate(zip(boundaries, boundaries[1:])):
jump = index % 5 == 4
if not cycle:
cycle = rng.permutation(len(lengths)).tolist()
if len(cycle) > 1 and cycle[0] == previous_shot:
cycle[0], cycle[1] = cycle[1], cycle[0]
shot = previous_shot if jump else cycle.pop(0)
previous_shot = shot
step = min(rate, (lengths[shot] - 1) / max(1, end - start - 1))
travel = round((end - start - 1) * step)
seek = int(rng.integers(0, lengths[shot] - travel))
if jump:
last = indices[-1] - offsets[shot]
if abs(seek - last) < 6:
seek = max((0, lengths[shot] - travel - 1), key=lambda value: abs(value - last))
indices.extend(offsets[shot] + seek + round(local * step) for local in range(end - start))
segments.append({"start_frame": start, "end_frame": end, "shot": shot + 1,
"source_start": seek, "playback_rate": step, "trigger": reasons[index],
"edit_type": "jump_cut" if jump else "shot_cut"})
return indices, segments
def draw_cursor(image, point, color, scale):
shape = np.array([[0, 0], [0, 23], [6, 17], [11, 28], [16, 25], [10, 15], [20, 15]], np.float32)
polygon = np.rint(shape * scale + point).astype(np.int32)
cv2.fillPoly(image, [polygon], (0.02, 0.025, 0.03), cv2.LINE_AA)
cv2.polylines(image, [polygon], True, (0.02, 0.025, 0.03), max(2, round(scale * 3)), cv2.LINE_AA)
cv2.fillPoly(image, [polygon], color, cv2.LINE_AA)
def order_windows(events, mode, ranks):
if mode == "random_on_snare":
return sorted(events, key=lambda e: ranks[e["cursor"]])
priority = {"voxel_on_top": 0, "edge_on_top": 1, "depth_on_top": 2}.get(mode)
if priority is not None:
return sorted(events, key=lambda e: e["effect"] == priority)
return events
def blend_reveal(region, layer, alpha, mode):
if mode == "screen":
layer = 1 - (1-region) * (1-layer)
elif mode == "add":
layer = (region + layer).clip(0,1)
region *= 1-alpha
region += layer * alpha
def reveal_fade(frame, event, fps, fade_in, fade_out):
drag_start = event["start"] + .12 * max(1, event["end"]-event["start"]-1)
attack = min(1, max(0, (frame-drag_start)/(fps*fade_in))) if fade_in else 1
release = min(1, (event["end"]-1-frame)/(fps*fade_out)) if fade_out else 1
return attack * max(0,release)
def cursor_plan(envelopes, count, seed, fps):
rng = np.random.default_rng(seed)
available = [0] * count
events = []
for frame in range(len(envelopes[0])):
hits = [band for band, values in enumerate(envelopes)
if values[frame] >= 0.5 and (frame == 0 or values[frame - 1] < 0.5)]
for band in hits:
free = [slot for slot, end in enumerate(available) if end <= frame]
if not free:
break
slot = int(rng.choice(free))
strength = envelopes[band][frame]
duration = int(rng.integers(max(4, round(fps * .3)), max(5, round(fps * .9))))
anchor = rng.uniform(.12, .88, 2)
direction = np.where(anchor > .5, -1, 1)
end = np.clip(anchor + direction * rng.uniform(.16, .48, 2) * (.7 + .3 * strength), .05, .95)
events.append({"start": frame, "end": frame + duration, "cursor": slot, "band": band,
"anchor": anchor.tolist(), "target": end.tolist(),
"approach": np.clip(anchor + rng.uniform(-.18, .18, 2), .02, .98).tolist(),
"bend": float(rng.uniform(-.12, .12)), "effect": int(rng.integers(0, 2))})
available[slot] = frame + duration + int(rng.integers(1, max(2, round(fps * .15))))
return events
def colorize_depth(depth, style):
depth = depth.clip(0,1)
if style == "grayscale":
return np.repeat(depth[:,:,None],3,axis=2)
if style == "false_color":
color = cv2.applyColorMap((depth*255).astype(np.uint8),cv2.COLORMAP_TURBO)[:,:,::-1].astype(np.float32)/255
color[depth == 0] = 0
return color
levels = np.floor(depth*12)/12
color = np.repeat(levels[:,:,None],3,axis=2)
edges = (depth*12 % 1 < .12) & (depth > 0)
color[edges] = (.2,.8,1)
return color
def assign_reveal_layers(events, values, seed):
rng = np.random.default_rng(seed+1729)
result = []
for event in events:
frame = event["start"]
activity = values["cursor_activity"][frame]
if activity < 1 and rng.random() >= activity:
continue
weights = np.array([values[k][frame] for k in ("voxel_weight","edge_weight","depth_weight")])
if weights[2] != 0 or weights[0] != weights[1]:
event["effect"] = int(rng.choice(3,p=weights/weights.sum()))
size = values["reveal_size"][frame]
if size != 1:
anchor = np.array(event["anchor"])
event["target"] = np.clip(anchor+(np.array(event["target"])-anchor)*size,.02,.98).tolist()
result.append(event)
return result
class FL_ScanAudioEdit:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"images": ("IMAGE",), "scan_images": ("IMAGE",), "normal_images": ("IMAGE",),
"kick_envelope": ("FL_AUDIO_ENVELOPE",), "snare_envelope": ("FL_AUDIO_ENVELOPE",),
"hihat_envelope": ("FL_AUDIO_ENVELOPE",),
"shot_lengths": ("STRING", {"default": "66,66,66,66"}),
"fps": ("INT", {"default": 24, "min": 1, "max": 120}),
"seed": ("INT", {"default": 73, "min": 0, "max": 2147483647}),
"min_cut_frames": ("INT", {"default": 10, "min": 2, "max": 120}),
"max_cut_frames": ("INT", {"default": 20, "min": 2, "max": 240}),
"playback_rate": ("FLOAT", {"default": 1.7, "min": 0.25, "max": 5, "step": 0.05}),
"cursor_count": ("INT", {"default": 2, "min": 0, "max": 3}),
"cursor_scale": ("FLOAT", {"default": 1.2, "min": 0.5, "max": 3, "step": 0.05}),
"reveal_strength": ("FLOAT", {"default": 1, "min": 0, "max": 1, "step": 0.01}),
"motion_strength": ("FLOAT", {"default": 0.8, "min": 0, "max": 2, "step": 0.05}),
}, "optional": {
"timing_mode": (["remix", "audio_locked"], {"default": "remix",
"tooltip": "Audio locked preserves chronological source frames and uses camera cuts instead of seeking or retiming."}),
}}
RETURN_TYPES = ("IMAGE", "IMAGE", "MASK", "STRING")
RETURN_NAMES = ("final", "edited_original", "reveal_mask", "edit_report")
FUNCTION = "render"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "FL drum envelopes drive jump cuts, punch-ins and cursor-drag layer reveals. All source banks must use identical frame order. Outputs last exactly as long as the envelopes. Maximum cut length caps holds between detected onsets."
def render(self, images, scan_images, normal_images, kick_envelope, snare_envelope, hihat_envelope,
shot_lengths, fps, seed, min_cut_frames, max_cut_frames, playback_rate, cursor_count,
cursor_scale, reveal_strength, motion_strength, timing_mode="remix"):
return self.render_mapped(images,scan_images,normal_images,kick_envelope,snare_envelope,hihat_envelope,
shot_lengths,fps,seed,min_cut_frames,max_cut_frames,playback_rate,cursor_count,cursor_scale,reveal_strength,
motion_strength,timing_mode)
def render_mapped(self, images, scan_images, normal_images, kick_envelope, snare_envelope, hihat_envelope,
shot_lengths, fps, seed, min_cut_frames, max_cut_frames, playback_rate, cursor_count,
cursor_scale, reveal_strength, motion_strength, timing_mode="remix", frame_values=None,
depth_images=None, depth_style="grayscale", window_order="newest_on_top", window_blend="normal",
window_fade_in=0, window_fade_out=0):
if images.shape != scan_images.shape or images.shape != normal_images.shape or images.shape[-1] != 3:
raise ValueError("Scan Edit needs matching RGB original, scan and projected-normal frame banks.")
lengths = [int(value.strip()) for value in shot_lengths.split(",")]
if sum(lengths) != len(images):
raise ValueError("Scan Edit shot lengths must add up to the source bank's frame count.")
envelopes = [load_audio_envelope(value) for value in (kick_envelope, snare_envelope, hihat_envelope)]
count = envelopes[0]["total_frames"]
if any(e["total_frames"] != count or not math.isclose(e["fps"], fps) for e in envelopes):
raise ValueError("Scan Edit envelopes must have equal frame counts and match the edit FPS.")
kick, snare, hat = [e["values"] for e in envelopes]
indices, segments = edit_plan(lengths, kick, snare, min_cut_frames, max_cut_frames, playback_rate, seed)
if timing_mode == "audio_locked":
if len(images) != count:
raise ValueError("Audio locked editing requires one chronological source frame per envelope frame.")
indices = list(range(count))
boundaries = {s["start_frame"]: s["trigger"] for s in segments}
offsets = [sum(lengths[:i]) for i in range(len(lengths))]
boundaries.update({offset: "authored_shot" for offset in offsets})
starts = sorted(boundaries)
segments = [{"start_frame": start, "end_frame": end,
"shot": max(i + 1 for i, offset in enumerate(offsets) if offset <= start),
"source_start": start, "playback_rate": 1, "trigger": boundaries[start],
"edit_type": "shot_cut" if start in offsets else "camera_cut"}
for start, end in zip(starts, starts[1:] + [count])]
elif timing_mode != "remix":
raise ValueError("Scan Edit timing mode must be remix or audio_locked.")
events = cursor_plan((kick, snare, hat), cursor_count, seed + 811, fps)
if frame_values is not None:
events = assign_reveal_layers(events,frame_values,seed)
height, width = images.shape[1:3]
final = np.empty((count, height, width, 3), np.float32)
original = np.empty_like(final)
masks = np.zeros((count, height, width), np.float32)
progress_bar = ProgressBar(count)
order_rng = random.Random(seed + 2909)
ranks = list(range(cursor_count))
order_rng.shuffle(ranks)
for cut, segment in enumerate(segments):
rng = np.random.default_rng(seed + cut * 997)
camera_zoom = float(rng.uniform(0, .22)) if timing_mode == "audio_locked" else 0
camera_angle = float(rng.uniform(-3, 3)) if timing_mode == "audio_locked" else 0
for frame in range(segment["start_frame"], segment["end_frame"]):
if frame_values is not None:
motion_strength,cursor_scale,reveal_strength = [frame_values[k][frame] for k in
("motion_strength","cursor_scale","reveal_strength")]
source = indices[frame]
original[frame] = images[source].cpu().float().numpy()
scan = scan_images[source].cpu().float().numpy()
normal = normal_images[source].cpu().float().numpy()
zoom = 1 + motion_strength * (camera_zoom + 0.16 * kick[frame] + 0.055 * snare[frame])
angle = motion_strength * (camera_angle + snare[frame] * (1.8 if cut % 2 else -1.8))
transform = cv2.getRotationMatrix2D((width / 2, height / 2), angle, zoom)
transform[0, 2] += math.sin(frame * 2.4) * hat[frame] * motion_strength * width * 0.004
composite = cv2.warpAffine(scan, transform, (width, height), borderMode=cv2.BORDER_REFLECT_101)
normal = cv2.warpAffine(normal, transform, (width, height), borderMode=cv2.BORDER_REFLECT_101)
effect = None
active = [e for e in events if e["start"] <= frame < e["end"]]
if window_order == "random_on_snare" and snare[frame] >= .5 and (frame == 0 or snare[frame-1] < .5):
order_rng.shuffle(ranks)
active = order_windows(active, window_order, ranks)
if any(e["effect"] == 1 for e in active):
gray = cv2.cvtColor((composite * 255).clip(0, 255).astype(np.uint8), cv2.COLOR_RGB2GRAY)
edges = cv2.Canny(gray, 55, 135).astype(np.float32) / 255
effect = composite[:, :, ::-1].copy() * 0.55
effect += edges[:, :, None] * np.array([0.2, 0.9, 1], np.float32)
effect = effect.clip(0, 1)
depth_layer = None
if any(e["effect"] == 2 for e in active):
depth_layer = colorize_depth(cv2.warpAffine(depth_images[source].cpu().numpy(),transform,(width,height),
borderMode=cv2.BORDER_REFLECT_101),depth_style)
for event in active:
anchor = np.array(event["anchor"]) * [width, height]
end = np.array(event["target"]) * [width, height]
age = (frame - event["start"]) / max(1, event["end"] - event["start"] - 1)
drag = min(1, max(0, (age - 0.12) / 0.58))
drag = 1 - (1 - drag) ** 2
tip = anchor + (end - anchor) * drag
if age < 0.12:
approach = np.array(event["approach"]) * [width, height]
tip = approach + (anchor - approach) * age / .12
else:
tip[0] += math.sin(drag * math.pi) * event["bend"] * width
tip = np.clip(tip, [0, 0], [width - 1, height - 1])
normals = event["effect"] == 0
is_depth = event["effect"] == 2
color = (1,.8,.2) if is_depth else ((1.0, 1.0, 1.0) if normals else (0.25, 1.0, 0.88))
x1, y1 = np.rint(np.minimum(anchor, tip)).astype(int)
x2, y2 = np.rint(np.maximum(anchor, tip)).astype(int)
if drag > 0 and x2 > x1 and y2 > y1:
layer = depth_layer if is_depth else (normal if normals else effect)
alpha = reveal_strength * (0.8 + 0.2 * snare[frame] if normals or is_depth else 0.55 + 0.45 * hat[frame])
opacity_key = ("voxel_opacity","edge_opacity","depth_opacity")[event["effect"]]
if frame_values is not None and opacity_key in frame_values:
alpha *= frame_values[opacity_key][frame]
alpha *= reveal_fade(frame,event,fps,window_fade_in,window_fade_out)
region = composite[y1:y2, x1:x2]
blend_reveal(region,layer[y1:y2, x1:x2],alpha,window_blend)
masks[frame, y1:y2, x1:x2] = 1 - (1 - masks[frame, y1:y2, x1:x2]) * (1 - alpha)
cv2.rectangle(composite, (x1, y1), (x2, y2), color, 1, cv2.LINE_AA)
label = "DEPTH / DRAG" if is_depth else ("NORMALS / DRAG" if normals else "EDGE SCAN / DRAG")
cv2.putText(composite, label, (x1, y1 - 7), cv2.FONT_HERSHEY_PLAIN, max(0.65, width / 750), color, 1, cv2.LINE_AA)
for x, y in ((x1, y1), (x2, y1), (x1, y2), (x2, y2)):
cv2.rectangle(composite, (x - 2, y - 2), (x + 2, y + 2), color, -1)
if 0.12 <= age < 0.35 or age > 0.9:
center = anchor if age < 0.35 else tip
radius = max(4, round(width * 0.015 * (1 + kick[frame])))
cv2.circle(composite, tuple(np.rint(center).astype(int)), radius, color, 1, cv2.LINE_AA)
draw_cursor(composite, tip, color, cursor_scale * max(0.5, width / 640))
final[frame] = composite.clip(0, 1)
progress_bar.update(1)
report = {"fps": fps, "frames": count, "duration": count / fps, "cuts": len(segments) - 1,
"audio_triggered_cuts": sum(s["trigger"] == "audio_onset" for s in segments),
"same_shot_jump_cuts": sum(s["edit_type"] == "jump_cut" for s in segments),
"timing_mode": timing_mode, "cursor_events": events,
"segments": segments, "source_indices": indices}
return torch.from_numpy(final), torch.from_numpy(original), torch.from_numpy(masks), json.dumps(report, indent=2)
+357
View File
@@ -0,0 +1,357 @@
"""Tracked detections and a depth-reprojected street-scan composite."""
import math
import cv2
import numpy as np
import torch
from comfy.utils import ProgressBar
class FL_ScanVideoDetections:
@classmethod
def INPUT_TYPES(cls):
return {"required": {"images": ("IMAGE",), "person_detector": ("SEGM_DETECTOR",),
"threshold": ("FLOAT", {"default": 0.35, "min": 0.05, "max": 1.0, "step": 0.01})},
"optional": {"face_detector": ("BBOX_DETECTOR",), "hand_detector": ("BBOX_DETECTOR",)}}
RETURN_TYPES = ("MASK", "FL_SCAN_TRACKS")
RETURN_NAMES = ("subject_masks", "detections")
FUNCTION = "detect"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "Run connected Impact detectors on each frame. Produces person masks and nearest-neighbor tracked boxes; no fabricated object labels."
def detect(self, images, person_detector, threshold, face_detector=None, hand_detector=None):
count, height, width = images.shape[:3]
masks = np.zeros((count, height, width), dtype=np.float32)
frames, previous, next_id = [], [], 1
progress = ProgressBar(count)
for frame in range(count):
detections = []
for label, detector in (("person", person_detector), ("face", face_detector), ("hand", hand_detector)):
if detector is None:
continue
_, segments = detector.detect(images[frame:frame+1], threshold, 0, 1.0, drop_size=8)
for segment in segments:
x1, y1, x2, y2 = map(int, segment.bbox)
detections.append({"label": label, "box": [x1, y1, x2, y2], "confidence": float(segment.confidence)})
if label == "person":
left, top, right, bottom = map(int, segment.crop_region)
mask = np.asarray(segment.cropped_mask, dtype=np.float32).squeeze()
masks[frame, top:bottom, left:right] = np.maximum(masks[frame, top:bottom, left:right], mask)
used = set()
for detection in detections:
box = detection["box"]
center = np.array([(box[0] + box[2]) / 2, (box[1] + box[3]) / 2])
candidates = [(np.linalg.norm(center - np.array([(old["box"][0] + old["box"][2]) / 2, (old["box"][1] + old["box"][3]) / 2])), old)
for old in previous if old["label"] == detection["label"] and old["id"] not in used]
distance, closest = min(candidates, key=lambda pair: pair[0]) if candidates else (float("inf"), None)
if distance < 0.18 * width:
detection["id"] = closest["id"]
else:
detection["id"] = next_id
next_id += 1
used.add(detection["id"])
frames.append(detections)
previous = detections
progress.update(1)
return torch.from_numpy(masks), {"width": width, "height": height, "frames": frames}
def scene_fragment(depth, subject, seed, raggedness):
height, width = depth.shape
rng = np.random.default_rng(seed)
coarse = cv2.resize(rng.uniform(-1, 1, (11, 11)).astype(np.float32), (width, height), interpolation=cv2.INTER_CUBIC)
chips = cv2.resize(rng.uniform(-1, 1, (48, 48)).astype(np.float32), (width, height), interpolation=cv2.INTER_NEAREST)
yy, xx = np.mgrid[:height, :width].astype(np.float32)
nx, ny = (xx - width / 2) / (width * 0.5), (yy - height / 2) / (height * 0.5)
radius = (np.abs(nx)**4 + np.abs(ny)**4)**0.25
field = radius + raggedness * (coarse * 0.55 + chips * 0.15) + (0.5 - depth) * 0.16
keep = (field < 0.87).astype(np.float32)
subject = cv2.dilate((subject > 0.4).astype(np.uint8), np.ones((5, 5), np.uint8))
return np.maximum(keep, subject).astype(bool)
def project_depth(depth, phase, orbit, relief, scale):
height, width = depth.shape
yy, xx = np.mgrid[:height, :width].astype(np.float32)
focal = width * 0.9
z = 1.5 + (1 - depth) * relief
x, y = (xx - width / 2) * z / focal, (yy - height / 2) * z / focal
angle = math.radians(orbit) * math.sin(phase * 2 * math.pi)
pivot = 1.5 + relief * 0.5
rotated_x = math.cos(angle) * x + math.sin(angle) * (z - pivot)
rotated_z = -math.sin(angle) * x + math.cos(angle) * (z - pivot) + pivot
px = rotated_x / rotated_z * focal * scale + width / 2
py = y / rotated_z * focal * scale + height / 2
return px, py, rotated_z
def project_parallax(depth, phase, orbit, relief, scale, strength, offset_x, offset_y, dolly, steady):
height, width = depth.shape
yy, xx = np.mgrid[:height, :width].astype(np.float32)
focal = width * .9
z = 1.5 + (1-depth)*relief
x,y = (xx-width/2)*z/focal, (yy-height/2)*z/focal
pivot = 1.5 + (1-steady)*relief
angle = math.radians(orbit)*math.sin(phase*math.tau)
rx = math.cos(angle)*x + math.sin(angle)*(z-pivot)
rz = -math.sin(angle)*x + math.cos(angle)*(z-pivot) + pivot
travel = dolly*strength
dx,dy = offset_x*strength*pivot, offset_y*strength*pivot
# Counter-shift and counter-zoom anchor the selected depth plane.
zoom = (pivot-travel)/pivot
denominator = np.maximum(.1,rz-travel)
px = ((rx-dx)/denominator*zoom + dx/pivot)*focal*scale+width/2
py = ((y-dy)/denominator*zoom + dy/pivot)*focal*scale+height/2
return px,py,denominator
def splat(rgb, keep, px, py, z, background):
height, width = keep.shape
channels = rgb.shape[-1]
xi, yi = np.rint(px).astype(np.int32), np.rint(py).astype(np.int32)
valid = keep & (xi >= 0) & (xi < width) & (yi >= 0) & (yi < height)
source = np.flatnonzero(valid)
target = yi.flat[source] * width + xi.flat[source]
order = np.lexsort((z.flat[source], target))
target, source = target[order], source[order]
_, first = np.unique(target, return_index=True)
target, source = target[first], source[first]
output = np.full((height * width, channels), background, np.float32)
output[target] = rgb.reshape(-1, channels)[source]
matte = np.zeros(height * width, np.uint8)
matte[target] = 255
return output.reshape(height, width, channels), matte.reshape(height, width)
def draw_box(image, box, color, text, opacity):
height, width = image.shape[:2]
x1, y1, x2, y2 = np.rint(box).astype(int)
x1, x2 = np.clip([x1, x2], 2, width - 3)
y1, y2 = np.clip([y1, y2], 14, height - 3)
if x2 - x1 < 4 or y2 - y1 < 4:
return
overlay = image.copy()
length = min(10, (x2 - x1) // 3, (y2 - y1) // 3)
for x, y, dx, dy in ((x1, y1, 1, 1), (x2, y1, -1, 1), (x1, y2, 1, -1), (x2, y2, -1, -1)):
cv2.line(overlay, (x, y), (x + dx * length, y), color, 1, cv2.LINE_AA)
cv2.line(overlay, (x, y), (x, y + dy * length), color, 1, cv2.LINE_AA)
text_width = min(width - x1 - 2, max(44, len(text) * 4))
cv2.rectangle(overlay, (x1, y1 - 11), (x1 + text_width, y1 - 1), color, -1)
cv2.putText(overlay, text, (x1 + 2, y1 - 3), cv2.FONT_HERSHEY_PLAIN, 0.55, (0.03, 0.04, 0.04), 1, cv2.LINE_AA)
image[:] = image * (1 - opacity) + overlay * opacity
def fill_projected_gaps(layers, matte):
height, width = matte.shape
points = cv2.findNonZero(matte)
if points is None:
return layers, matte
hull = np.zeros_like(matte)
cv2.fillConvexPoly(hull, cv2.convexHull(points), 255)
holes = (hull > 0) & (matte == 0)
if holes.any():
_, labels = cv2.distanceTransformWithLabels((matte == 0).astype(np.uint8), cv2.DIST_L2, 5,
labelType=cv2.DIST_LABEL_PIXEL)
layers[holes] = layers[matte > 0][labels[holes] - 1]
layers[hull == 0] = 0
return layers, hull
def digital_layers(layers, matte, phase, relief, count=4, spacing=1, x=1, y=1, rotation=0, opacity=1, palette="cobalt"):
height, width = matte.shape
if not matte.any():
return layers,matte,layers[:,:,:3].copy()
layers,hull = fill_projected_gaps(layers,matte)
composite = np.zeros((height, width, 3), np.float32)
step = max(2, round(width * (.009 + .004 * relief))) * spacing
bright = {"cobalt": (.035,.02,.9), "cyan": (.02,.7,.85), "magenta": (.8,.02,.6), "mono": (.65,.65,.7)}[palette]
for level in range(count if opacity > 0 else 0, 0, -1):
shift = cv2.getRotationMatrix2D((width/2,height/2), level * rotation, 1).astype(np.float32)
shift[0,2] += level * step * x
shift[1,2] += level * step * y * (.65 + .2 * math.sin(phase * math.tau))
plate = cv2.warpAffine(hull, shift, (width, height), flags=cv2.INTER_NEAREST)
color = bright if level % 2 == 0 else (.04, .04, .055)
target = composite if opacity == 1 else composite.copy()
target[plate > 0] = color
contours, _ = cv2.findContours(plate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(target, contours, -1, (.85, .9, 1), max(1, width // 640), cv2.LINE_8)
if opacity != 1:
cv2.addWeighted(target, opacity, composite, 1-opacity, 0, dst=composite)
composite[hull > 0] = layers[:, :, :3][hull > 0]
contours, _ = cv2.findContours(hull, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(composite, contours, -1, (1, 1, 1), max(1, width // 320), cv2.LINE_8)
return layers, hull, composite
class FL_StreetScanComposite:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"images": ("IMAGE",), "depth": ("IMAGE",), "normals": ("IMAGE",),
"subject_masks": ("MASK",), "detections": ("FL_SCAN_TRACKS",),
"fps": ("FLOAT", {"default": 24, "min": 1, "max": 120}),
"seed": ("INT", {"default": 41, "min": 0, "max": 2147483647}),
"orbit_degrees": ("FLOAT", {"default": 9, "min": 0, "max": 25, "step": 0.5}),
"depth_relief": ("FLOAT", {"default": 1.4, "min": 0, "max": 3, "step": 0.05}),
"scene_scale": ("FLOAT", {"default": 0.88, "min": 0.5, "max": 1.0, "step": 0.01}),
"raggedness": ("FLOAT", {"default": 0.42, "min": 0, "max": 1, "step": 0.01}),
"normal_mix": ("FLOAT", {"default": 0.85, "min": 0, "max": 1, "step": 0.01}),
"hud_opacity": ("FLOAT", {"default": 0.85, "min": 0, "max": 1, "step": 0.01}),
"echo_strength": ("FLOAT", {"default": 0.45, "min": 0, "max": 1, "step": 0.01}),
}, "optional": {"pose_keypoints": ("POSE_KEYPOINT",),
"pose_opacity": ("FLOAT", {"default": 0.65, "min": 0, "max": 1, "step": 0.01}),
"edge_style": (["fragment", "digital_layers"], {"default": "fragment",
"tooltip": "Digital layers keeps the full scene on stacked cobalt panels. Raggedness and temporal echoes apply only to fragment mode."})}}
RETURN_TYPES = ("IMAGE", "MASK", "IMAGE")
RETURN_NAMES = ("composite", "scene_matte", "projected_normals")
FUNCTION = "render"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "2.5D depth reprojection with a fragmented boundary or clean stacked digital panels, masked normal-map flashes and detection HUD. Run separately per shot to reset temporal state. This is not a reconstructed multi-view 3D mesh."
def render(self, images, depth, normals, subject_masks, detections, fps, seed, orbit_degrees, depth_relief,
scene_scale, raggedness, normal_mix, hud_opacity, echo_strength, pose_keypoints=None, pose_opacity=0.65,
edge_style="fragment"):
return self.render_layers(images,depth,normals,subject_masks,detections,fps,seed,orbit_degrees,depth_relief,
scene_scale,raggedness,normal_mix,hud_opacity,echo_strength,pose_keypoints,pose_opacity,edge_style)[:3]
def render_layers(self, images, depth, normals, subject_masks, detections, fps, seed, orbit_degrees, depth_relief,
scene_scale, raggedness, normal_mix, hud_opacity, echo_strength, pose_keypoints=None, pose_opacity=0.65,
edge_style="fragment", frame_values=None, motion_mode="current", parallax_scope="whole_scene",
stack_count=4, stack_palette="cobalt"):
if edge_style not in ("fragment", "digital_layers"):
raise ValueError("Street Scan edge style must be fragment or digital_layers.")
count, height, width = images.shape[:3]
if depth.shape[:3] != images.shape[:3] or normals.shape[:3] != images.shape[:3] or tuple(subject_masks.shape) != (count, height, width):
raise ValueError("Street Scan requires one aligned depth, normal and subject mask for every source frame.")
if len(detections["frames"]) != count or (detections["height"], detections["width"]) != (height, width):
raise ValueError("Street Scan detections must come from the same frame batch and resolution.")
if pose_keypoints is not None and len(pose_keypoints) != count:
raise ValueError("Street Scan needs one DWPose keypoint frame for every source frame.")
output = np.empty((count, height, width, 3), np.float32)
projected_normals = np.empty_like(output)
mattes = np.empty((count, height, width), np.float32)
depth_output = np.empty((count,height,width),np.float32) if frame_values is not None else None
history = []
progress = ProgressBar(count)
colors = {"person": (0.45, 1.0, 0.07), "face": (0.28, 0.48, 1.0), "hand": (1.0, 0.82, 0.12)}
previous_depth = None
previous_gray, anchors = None, None
for frame in range(count):
if frame_values is not None:
orbit_degrees,depth_relief,scene_scale,normal_mix,hud_opacity,pose_opacity = [frame_values[k][frame] for k in
("orbit_degrees","depth_relief","scene_scale","normal_mix","hud_opacity","pose_opacity")]
rgb = images[frame, :, :, :3].cpu().float().numpy().copy()
d = cv2.GaussianBlur(depth[frame, :, :, 0].cpu().float().numpy(), (0, 0), 1.5)
if previous_depth is not None:
d = d * 0.85 + previous_depth * 0.15
previous_depth = d
normal = normals[frame, :, :, :3].cpu().float().numpy()
subject = subject_masks[frame].cpu().float().numpy()
keep = np.ones((height, width), bool) if edge_style == "digital_layers" else scene_fragment(d, subject, seed, raggedness)
phase = frame / max(1, count - 1)
pulse = (frame + seed) % max(1, round(fps * 1.2))
mix = normal_mix if pulse < 5 else normal_mix * 0.06
rgb = rgb * (1 - subject[:, :, None] * mix) + normal * subject[:, :, None] * mix
normal_window = None
if 8 <= pulse < 19 and normal_mix > 0:
left = 0.12 if seed % 2 else 0.6
x1, x2 = round(width * left), round(width * (left + 0.26))
y1, y2 = round(height * 0.19), round(height * 0.52)
normal_window = (x1, y1, x2, y2)
alpha = (1 - subject[y1:y2, x1:x2, None]) * normal_mix
rgb[y1:y2, x1:x2] = rgb[y1:y2, x1:x2] * (1 - alpha) + normal[y1:y2, x1:x2] * alpha
px, py, z = project_depth(d, phase, orbit_degrees, depth_relief, scene_scale)
alternate = None
if motion_mode == "depth_parallax":
alternate = project_parallax(d,phase,orbit_degrees,depth_relief,scene_scale,
*[frame_values[k][frame] for k in ("parallax_strength","offset_x","offset_y","dolly","steady_depth")])
if parallax_scope == "whole_scene":
px,py,z = alternate
packed = np.concatenate((rgb,normal,d[:,:,None]),axis=-1) if depth_output is not None else np.concatenate((rgb,normal),axis=-1)
background = (0.18,0.19,0.20)*2 + ((0,) if depth_output is not None else ())
layers, matte = splat(packed, keep, px, py, z, background)
if edge_style == "digital_layers":
stack = {key.removeprefix("stack_"): frame_values[key][frame] for key in
("stack_spacing","stack_x","stack_y","stack_rotation","stack_opacity") if frame_values is not None and key in frame_values}
layers, matte, digital = digital_layers(layers, matte, phase, depth_relief, count=stack_count, palette=stack_palette, **stack)
projected = layers[:, :, :3].copy()
projected_normals[frame] = layers[:, :, 3:6]
if depth_output is not None:
depth_output[frame] = layers[:,:,6]
if alternate is not None and parallax_scope == "reveals_only":
reveal_layers,reveal_matte = splat(np.concatenate((normal,d[:,:,None]),axis=-1),keep,*alternate,(0,0,0,0))
reveal_layers,_ = fill_projected_gaps(reveal_layers,reveal_matte)
projected_normals[frame] = reveal_layers[:,:,:3]
depth_output[frame] = reveal_layers[:,:,3]
composite = digital if edge_style == "digital_layers" else projected.copy()
for age, (old, old_matte) in enumerate(reversed(history) if edge_style == "fragment" else (), 1):
transform = np.float32([[1, 0, age * 3], [0, 1, age * 4]])
ghost = cv2.warpAffine(old, transform, (width, height), borderValue=(0.18, 0.19, 0.20))
ghost_mask = cv2.warpAffine(old_matte, transform, (width, height))
alpha = ((matte == 0) & (ghost_mask > 128)).astype(np.float32)[:, :, None] * echo_strength / age
composite = composite * (1 - alpha) + ghost * alpha
if edge_style == "fragment":
history.append((projected, matte))
history = history[-2:]
centers = []
if normal_window is not None:
x1, y1, x2, y2 = normal_window
box = [px[y1, x1], py[y1, x1], px[y2, x2], py[y2, x2]]
draw_box(composite, box, (1.0, 0.52, 0.16), "normal_field", hud_opacity)
for detection in detections["frames"][frame]:
x1, y1, x2, y2 = detection["box"]
xs = np.clip([x1, x2 - 1, x1, x2 - 1], 0, width - 1)
ys = np.clip([y1, y1, y2 - 1, y2 - 1], 0, height - 1)
box = [float(px[ys, xs].min()), float(py[ys, xs].min()), float(px[ys, xs].max()), float(py[ys, xs].max())]
color = colors[detection["label"]]
text = f"{detection['label']}_{detection['id']:02} {detection['confidence']:.2f}"
draw_box(composite, box, color, text, hud_opacity)
centers.append((int((box[0] + box[2]) / 2), int((box[1] + box[3]) / 2)))
if pose_keypoints is not None and pulse >= 19 and pose_opacity > 0:
pose_frame = pose_keypoints[frame]
overlay = composite.copy()
for person in pose_frame["people"]:
for part in ("pose_keypoints_2d", "hand_left_keypoints_2d", "hand_right_keypoints_2d"):
values = person.get(part)
if not values:
continue
points = np.asarray(values, dtype=np.float32).reshape(-1, 3)
xs = np.clip(np.rint(points[:, 0] * width / pose_frame["canvas_width"]).astype(int), 0, width - 1)
ys = np.clip(np.rint(points[:, 1] * height / pose_frame["canvas_height"]).astype(int), 0, height - 1)
projected_points = np.stack([px[ys, xs], py[ys, xs]], axis=-1).astype(int)
edges = [(1, 2), (2, 3), (3, 4), (1, 5), (5, 6), (6, 7), (1, 8), (8, 9), (9, 10), (1, 11), (11, 12), (12, 13), (1, 0), (0, 14), (0, 15)] if part == "pose_keypoints_2d" else [(0 if index % 4 == 1 else index - 1, index) for index in range(1, 21)]
for index, (a, b) in enumerate(edges):
if b >= len(points) or a >= len(points) or min(points[a, 2], points[b, 2]) < 0.3:
continue
color = ((0.2, 1.0, 0.65), (1.0, 0.75, 0.2), (0.55, 0.45, 1.0))[index % 3]
cv2.line(overlay, tuple(projected_points[a]), tuple(projected_points[b]), color, 1, cv2.LINE_AA)
cv2.circle(overlay, tuple(projected_points[b]), 2, color, -1, cv2.LINE_AA)
composite = composite * (1 - pose_opacity) + overlay * pose_opacity
source_rgb = images[frame, :, :, :3].cpu().float().numpy()
gray = cv2.cvtColor((source_rgb * 255).clip(0, 255).astype(np.uint8), cv2.COLOR_RGB2GRAY)
if previous_gray is not None and anchors is not None:
moved, valid, error = cv2.calcOpticalFlowPyrLK(previous_gray, gray, anchors, None, winSize=(21, 21), maxLevel=2)
anchors = moved[(valid[:, 0] == 1) & (error[:, 0] < 30)]
if not len(anchors):
anchors = None
if anchors is None or len(anchors) < 3:
anchors = cv2.goodFeaturesToTrack(gray, 5, 0.05, width * 0.16, mask=(keep & (subject < 0.2)).astype(np.uint8) * 255)
previous_gray = gray
if anchors is not None and hud_opacity > 0:
overlay = composite.copy()
for index, point in enumerate(anchors[:, 0]):
x, y = np.clip(np.rint(point).astype(int), [0, 0], [width - 1, height - 1])
target = (int(px[y, x]), int(py[y, x]))
if centers:
cv2.line(overlay, centers[index % len(centers)], target, (0.47, 0.73, 0.27), 1, cv2.LINE_AA)
cv2.drawMarker(overlay, target, (0.7, 0.95, 0.2), cv2.MARKER_CROSS, 7, 1)
cv2.putText(overlay, f"Z_REL {d[y,x]:.2f}", (target[0] + 4, target[1] - 4), cv2.FONT_HERSHEY_PLAIN, 0.5, (0.75, 0.95, 0.3), 1, cv2.LINE_AA)
composite = composite * (1 - hud_opacity) + overlay * hud_opacity
output[frame] = composite.clip(0, 1)
mattes[frame] = matte / 255.0
progress.update(1)
return (torch.from_numpy(output), torch.from_numpy(mattes), torch.from_numpy(projected_normals),
torch.from_numpy(depth_output) if depth_output is not None else None)
+88
View File
@@ -0,0 +1,88 @@
"""Screen-aligned normal-color cuboids with animated depth relief."""
import math
import cv2
import numpy as np
import torch
from comfy.utils import ProgressBar
class FL_VoxelNormalRelief:
@classmethod
def INPUT_TYPES(cls):
return {"required": {
"normals": ("IMAGE",), "depth": ("IMAGE",),
"cube_size": ("INT", {"default": 12, "min": 4, "max": 64}),
"relief": ("FLOAT", {"default": 0.65, "min": 0, "max": 2, "step": 0.05}),
"animation": ("FLOAT", {"default": 0.18, "min": 0, "max": 1, "step": 0.01}),
"speed": ("FLOAT", {"default": 0.7, "min": 0, "max": 5, "step": 0.05}),
"fps": ("FLOAT", {"default": 24, "min": 1, "max": 120}),
"seed": ("INT", {"default": 41, "min": 0, "max": 2147483647}),
}}
RETURN_TYPES = ("IMAGE",)
RETURN_NAMES = ("voxel_normals",)
FUNCTION = "render"
CATEGORY = "🏵️Fill Nodes/VFX"
DESCRIPTION = "Renders normal-colored cuboids at their source-image positions. Depth drives height; a smooth seeded wave adds subtle motion. Connect before the scan compositor's depth projection. This is a 2.5D image effect, not a voxel mesh."
def render(self, normals, depth, cube_size, relief, animation, speed, fps, seed):
return self.render_animated(normals, depth, cube_size, relief, animation, speed, fps, seed)
def render_animated(self, normals, depth, cube_size, relief, animation, speed, fps, seed, frame_values=None):
if normals.ndim != 4 or normals.shape[-1] != 3 or depth.ndim != 4 or depth.shape[:3] != normals.shape[:3] or depth.shape[-1] < 1:
raise ValueError("Voxel Normal Relief needs aligned RGB normals and depth frames at the same resolution.")
if cube_size < 4 or fps <= 0 or not all(math.isfinite(v) and v >= 0 for v in (relief, animation, speed, fps)):
raise ValueError("Voxel Normal Relief requires cube size >= 4, positive FPS and finite nonnegative controls.")
count, height, width = normals.shape[:3]
output = np.empty((count, height, width, 3), np.float32)
rows, cols = math.ceil(height / cube_size), math.ceil(width / cube_size)
gap = max(1, round(cube_size * .22))
rng = np.random.default_rng(seed)
phases = rng.uniform(0, math.tau, (rows, cols))
yy, xx = np.mgrid[:rows, :cols]
x, y = xx * cube_size, yy * cube_size
right, bottom = np.minimum(width, x + cube_size - gap), np.minimum(height, y + cube_size - gap)
bases = np.stack((np.stack((x, y), -1), np.stack((right, y), -1),
np.stack((right, bottom), -1), np.stack((x, bottom), -1)), -2).astype(np.int32)
progress = ProgressBar(count)
phase = 0.0
for frame in range(count):
normal = normals[frame].cpu().float().numpy()
d = depth[frame, :, :, 0].cpu().float().numpy()
normal = cv2.copyMakeBorder(normal, 0, rows * cube_size - height, 0, cols * cube_size - width, cv2.BORDER_REPLICATE)
d = cv2.copyMakeBorder(d, 0, rows * cube_size - height, 0, cols * cube_size - width, cv2.BORDER_REPLICATE)
colors = cv2.resize(normal, (cols, rows), interpolation=cv2.INTER_AREA).clip(0, 1)
levels = cv2.resize(d, (cols, rows), interpolation=cv2.INTER_AREA).clip(0, 1)
if frame_values is None:
phase = frame / fps * speed * math.tau
else:
relief = frame_values["relief"][frame]
animation = frame_values["animation"][frame]
wave = np.sin(phase + xx * .42 + yy * .31 + phases * .25)
heights = cube_size * relief * np.maximum(.05, .3 + .6 * levels + animation * wave)
if frame_values is not None:
phase += frame_values["speed"][frame] / fps * math.tau
image = np.zeros((height, width, 3), np.float32)
offsets = np.stack((-heights * .45, -heights * .65), -1)[..., None, :]
tops = np.rint(bases + offsets).astype(np.int32)
sides = np.stack((tops[..., 1, :], bases[..., 1, :], bases[..., 2, :], tops[..., 2, :]), -2).reshape(-1, 4, 2)
fronts = np.stack((tops[..., 3, :], tops[..., 2, :], bases[..., 2, :], bases[..., 3, :]), -2).reshape(-1, 4, 2)
tops = tops.reshape(-1, 4, 2)
colors = colors.reshape(-1, 3)
side_colors, front_colors = (colors * .48).tolist(), (colors * .68).tolist()
line_colors, highlight_colors = (colors * .38).tolist(), (colors * .75 + .2).tolist()
colors = colors.tolist()
edges = tops[:, :2].tolist()
# Front rows and right-hand cells cover the sides of cubes behind them.
for top, side, front, edge, color, side_color, front_color, line_color, highlight in zip(
tops, sides, fronts, edges, colors, side_colors, front_colors, line_colors, highlight_colors):
cv2.fillConvexPoly(image, side, side_color, cv2.LINE_8)
cv2.fillConvexPoly(image, front, front_color, cv2.LINE_8)
cv2.fillConvexPoly(image, top, color, cv2.LINE_8)
cv2.polylines(image, [top], True, line_color, 1, cv2.LINE_8)
cv2.line(image, edge[0], edge[1], highlight, 1, cv2.LINE_8)
output[frame] = image.clip(0, 1)
progress.update(1)
return (torch.from_numpy(output),)
+74
View File
@@ -0,0 +1,74 @@
import asyncio
import json
import os
from pathlib import Path
import re
import threading
import uuid
from safetensors import safe_open
from safetensors.torch import save_file
import folder_paths
from comfy_execution.cache_provider import CacheProvider, CacheValue
class PosterLayerCache(CacheProvider):
"""Exact RGBA recovery for poster assets evicted by ComfyUI's RAM cache."""
def __init__(self, directory=None, maximum_bytes=1024**3):
self.directory = Path(directory) if directory is not None else Path(folder_paths.get_user_directory()) / "__cache" / "fl_poster_layers_v1"
self.maximum_bytes = maximum_bytes
self.lock = threading.Lock()
def should_cache(self, context, value=None):
return context.class_type == "FL_PosterLayerAsset" and re.fullmatch(r"[0-9a-f]{64}", context.cache_key_hash) is not None
async def on_lookup(self, context):
if not self.should_cache(context):
return None
return await asyncio.to_thread(self._lookup, context.cache_key_hash)
def _lookup(self, key):
path = self.directory / (key + ".safetensors")
with self.lock:
if not path.is_file():
return None
with safe_open(path, framework="pt", device="cpu") as data:
asset = json.loads(data.metadata()["asset"])
# Release the file mapping so Windows can evict old cache files.
asset["image"] = data.get_tensor("image").clone()
os.utime(path, None)
return CacheValue(outputs=[[asset]])
async def on_store(self, context, value):
if not self.should_cache(context):
return
# ComfyUI caches a list of mapped values for each output socket.
asset = value.outputs[0][0]
await asyncio.to_thread(self._store, context.cache_key_hash, asset)
def _store(self, key, asset):
with self.lock:
self._write(key, asset)
def _write(self, key, asset):
self.directory.mkdir(parents=True, exist_ok=True)
image = asset["image"].detach().cpu().contiguous()
if image.numel() * image.element_size() > self.maximum_bytes:
return
metadata = {k: v for k, v in asset.items() if k != "image"}
target = self.directory / (key + ".safetensors")
temporary = self.directory / (key + "." + uuid.uuid4().hex + ".tmp")
save_file({"image": image}, str(temporary), metadata={"asset": json.dumps(metadata)})
os.replace(temporary, target)
files = sorted((p for p in self.directory.iterdir() if re.fullmatch(r"[0-9a-f]{64}\.safetensors", p.name)), key=lambda p: p.stat().st_mtime)
total = sum(p.stat().st_size for p in files)
for path in files:
if total <= self.maximum_bytes:
break
if path == target:
continue
size = path.stat().st_size
path.unlink()
total -= size
+69
View File
@@ -0,0 +1,69 @@
import math
TARGETS = {
"stack_spacing": (0,4), "stack_x": (-2,2), "stack_y": (-2,2),
"stack_rotation": (-15,15), "stack_opacity": (0,1),
"voxel_opacity": (0,1), "edge_opacity": (0,1), "depth_opacity": (0,1),
"relief": (0,2), "animation": (0,1), "speed": (0,5),
"motion_strength": (0,2), "orbit_degrees": (0,25), "depth_relief": (0,3),
"scene_scale": (.5,1), "normal_mix": (0,1), "hud_opacity": (0,1), "pose_opacity": (0,1),
"parallax_strength": (0,2), "offset_x": (-.2,.2), "offset_y": (-.2,.2),
"dolly": (-.3,.3), "steady_depth": (0,1),
"cursor_activity": (0,1), "reveal_size": (.25,1.75), "cursor_scale": (.5,3), "reveal_strength": (0,1),
"voxel_weight": (0,100), "edge_weight": (0,100), "depth_weight": (0,100),
"brightness": (0,3), "saturation": (0,3), "glow": (0,2),
}
def compile_mappings(settings, envelopes, fps, lengths):
count = sum(lengths)
curves = {key: [settings[key]] * count for key in TARGETS if key in settings}
# These are the original three audio reactions, expressed as absolute values.
curves["brightness"] = [settings["base_brightness"] + v * settings["brightness_intensity"] for v in envelopes[1]["values"]]
curves["saturation"] = [max(0,settings["base_saturation"] + v * settings["saturation_intensity"]) for v in envelopes[0]["values"]]
curves["glow"] = [settings["glow_intensity"] + v * settings["envelope_intensity"] for v in envelopes[0]["values"]]
rows = settings["audio_mappings"]
if not isinstance(rows,list):
raise ValueError("Audio mappings must be a list.")
occupied = {}
boundaries = set()
offset = 0
for length in lengths:
boundaries.add(offset)
offset += length
evaluated = []
for index,row in enumerate(rows):
if not isinstance(row,dict) or row.keys() - {"enabled","source","target","minimum","maximum","start_frame","end_frame","invert","smoothing"}:
raise ValueError(f"Audio mapping {index+1}: unknown fields.")
if not isinstance(row.get("enabled",True),bool) or not isinstance(row.get("invert",False),bool):
raise ValueError(f"Audio mapping {index+1}: enabled and invert must be booleans.")
if not row.get("enabled",True):
continue
target,source = row.get("target"),row.get("source")
if target not in TARGETS or type(source) is not int or not 0 <= source < 3:
raise ValueError(f"Audio mapping {index+1}: select a supported parameter and envelope.")
start,end = row.get("start_frame",0),row.get("end_frame")
end = count if end is None else end
if type(start) is not int or type(end) is not int or not 0 <= start < end <= count:
raise ValueError(f"Audio mapping {index+1}: range must fit the video (end is exclusive).")
low,high = TARGETS[target]
minimum,maximum,smoothing = row.get("minimum"),row.get("maximum"),row.get("smoothing",0)
if any(type(v) not in (int,float) or not math.isfinite(v) for v in (minimum,maximum,smoothing)) or not low <= minimum <= maximum <= high or not 0 <= smoothing <= 2:
raise ValueError(f"Audio mapping {index+1}: invalid range or smoothing; {target} supports {low} to {high}.")
if any(start < b and end > a for a,b in occupied.get(target,[])):
raise ValueError(f"Audio mappings for {target} overlap. Use non-overlapping ranges.")
occupied.setdefault(target,[]).append((start,end))
coefficient = 1 if smoothing == 0 else 1-math.exp(-1/(fps*smoothing))
previous = None
for frame in range(start,end):
value = envelopes[source]["values"][frame]
if row.get("invert",False):
value = 1-value
value = minimum + (maximum-minimum)*value
previous = value if previous is None or frame in boundaries else previous + coefficient*(value-previous)
curves[target][frame] = previous
evaluated.append({**row,"start_frame":start,"end_frame":end})
if any(sum(curves[k][i] for k in ("voxel_weight","edge_weight","depth_weight")) <= 0 for i in range(count)):
raise ValueError("Reveal layer weights must have a positive total at every frame.")
return curves,evaluated
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui_fill-nodes"
description = "Fill-Nodes is a versatile collection of custom nodes for ComfyUI that extends functionality across multiple domains. Features include advanced image processing (pixelation, slicing, masking), visual effects generation (glitch, halftone, pixel art), comprehensive file handling (PDF creation/extraction, Google Drive integration), AI model interfaces (GPT, DALL-E, Hugging Face), utility nodes for workflow enhancement, and specialized tools for video processing, captioning, and batch operations. The pack provides both practical workflow solutions and creative tools within a unified node collection."
version = "2.30.0"
version = "2.31.0"
license = {file = "LICENSE"}
dependencies = [
"librosa",
+1
View File
@@ -1,4 +1,5 @@
from . import audio_timeline
from . import prompt_storyboards
from . import load_video
from . import video_combine
+117
View File
@@ -0,0 +1,117 @@
import asyncio
import io
from PIL import Image, ImageOps
from aiohttp import web
from server import PromptServer
import execution
from ..nodes.audio.prompt_storyboards import storyboard_store, storyboard_graph, storyboard_batch_graph, extract_panels
from ..nodes.audio.prompt_references import reference_path
PREFIX = "/fl/audio-prompt-timeline/storyboards"
def thumbnail_bytes(path):
with Image.open(path) as source:
source.thumbnail((256, 256))
image = ImageOps.exif_transpose(source).convert("RGB")
output = io.BytesIO()
image.save(output, format="WEBP", quality=78)
return output.getvalue()
@PromptServer.instance.routes.get(PREFIX + "/thumbnail")
async def storyboard_thumbnail(request):
try:
path = reference_path(dict(request.query))
stat = path.stat()
etag = f'"{stat.st_mtime_ns}-{stat.st_size}-256"'
headers = {"ETag": etag, "Cache-Control": "private, max-age=0, must-revalidate"}
if request.headers.get("If-None-Match") == etag:
return web.Response(status=304, headers=headers)
return web.Response(body=await asyncio.to_thread(thumbnail_bytes, path), content_type="image/webp", headers=headers)
except (ValueError, OSError) as error:
return web.json_response({"error": str(error)}, status=400)
@PromptServer.instance.routes.get(PREFIX)
async def list_storyboards(request):
return web.json_response(await asyncio.to_thread(storyboard_store.list, request.query.get("scheduler_id", "")))
@PromptServer.instance.routes.post(PREFIX)
async def create_storyboard(request):
try:
result = await asyncio.to_thread(storyboard_store.create, await request.json())
return web.json_response(result)
except (ValueError, TypeError, KeyError) as error:
return web.json_response({"error": str(error)}, status=400)
@PromptServer.instance.routes.post(PREFIX + "/{job_id}/{action}")
async def storyboard_action(request):
try:
job_id, action = request.match_info["job_id"], request.match_info["action"]
if job_id == "batch" and action == "submit":
ids = (await request.json()).get("job_ids")
if not isinstance(ids, list) or not ids or any(not isinstance(i,str) for i in ids) or len(set(ids)) != len(ids):
raise ValueError("Submit a nonempty list of distinct storyboard jobs.")
jobs = [await asyncio.to_thread(storyboard_store.get, i) for i in ids]
if len({j["spec"]["scheduler_id"] for j in jobs}) != 1:
raise ValueError("A storyboard batch must belong to one scheduler.")
graph = storyboard_batch_graph(jobs)
valid, error, _outputs, node_errors = await execution.validate_prompt("storyboard_batch", graph, None)
if not valid:
return web.json_response({"error": "Storyboard batch validation failed", "details": error, "node_errors": node_errors}, status=400)
await asyncio.to_thread(storyboard_store.claim_batch, ids)
return web.json_response({"graph": graph})
job = await asyncio.to_thread(storyboard_store.get, job_id)
if action in {"validate", "submit"}:
graph = storyboard_graph(job_id, job["spec"])
valid, error, _outputs, node_errors = await execution.validate_prompt(job_id, graph, None)
if not valid:
return web.json_response({"error": "Storyboard graph validation failed", "details": error, "node_errors": node_errors}, status=400)
if action == "validate":
return web.json_response({"valid": True})
graph = await asyncio.to_thread(storyboard_store.claim, job_id)
return web.json_response({"graph": graph})
if action == "receipt":
body = await request.json()
prompt_id = body.get("prompt_id")
if job["state"] not in {"submitted", "unknown"} or not isinstance(prompt_id, str) or len(prompt_id) > 128:
raise ValueError("Invalid storyboard queue receipt.")
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "submitted", {"prompt_id": prompt_id}))
if action == "refresh":
if job["state"] in {"proposed", "cancelled", "complete"}:
return web.json_response(job)
history = PromptServer.instance.prompt_queue.get_history()
for prompt_id, entry in history.items():
images = [image for output in entry.get("outputs", {}).values() for image in output.get("images", [])]
source = next((image for image in images if image.get("subfolder", "").replace("\\", "/") == f"fl-storyboards/{job_id}"), None)
if source:
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "complete", {**job["result"], "source": source, "prompt_id": prompt_id}))
prompt_id = job["result"].get("prompt_id")
if prompt_id in history:
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "failed", {**job["result"], "error": "No contact sheet was saved. Check ComfyUI history; charges may apply."}))
running, pending = PromptServer.instance.prompt_queue.get_current_queue()
for item in running + pending:
graph = item[2]
if any(node.get("inputs", {}).get("filename_prefix") == f"fl-storyboards/{job_id}/sheet" for node in graph.values()):
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "submitted", {"prompt_id": item[1]}))
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "unknown", {**job["result"], "error": "No queue or history record found. Check saved outputs before submitting a new paid job."}))
if action == "extract":
if job["state"] != "complete" or not job["result"].get("source"):
raise ValueError("Wait for a completed contact sheet before extracting panels.")
body = await request.json()
result = await asyncio.to_thread(extract_panels, job, job["result"]["source"], body.get("bounds"))
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "complete", result))
if action == "cancel":
if job["state"] != "proposed":
raise ValueError("Submitted jobs may already be billed. Use ComfyUI's queue to cancel pending work.")
return web.json_response(await asyncio.to_thread(storyboard_store.update, job_id, "cancelled", {}))
raise ValueError("Unknown storyboard action.")
except (ValueError, TypeError, KeyError) as error:
return web.json_response({"error": str(error)}, status=400)
+44
View File
@@ -0,0 +1,44 @@
export async function testControls() {
const {createScanControls}=await import(new URL('/extensions/ComfyUI_Fill-Nodes/nodes/vfx/scan_controls.js',location.origin));
const {createAudioMappingPanel}=await import(new URL('/extensions/ComfyUI_Fill-Nodes/nodes/vfx/scan_audio_mapping.js',location.origin));
const info=(await (await fetch('/object_info/FL_InteractiveScanFX')).json()).FL_InteractiveScanFX.input.required;
const settings={value:info.advanced_settings[1].default};
const node={widgets:Object.entries(info).filter(([k,v])=>v[1]?.default!==undefined&&k!=='advanced_settings').map(([name,v])=>({name,value:v[1].default,options:v[1]})),inputs:[],graph:{setDirtyCanvas(){}}};
const panel=createAudioMappingPanel({settings,targets:info.advanced_settings[1].scan_mapping_targets,onChange(){},onSeek(){}});
panel.update({frames:192,envelopes:Array.from({length:3},()=>Array(192).fill(0)),segments:[],curves:{}},0);
const ui=createScanControls({node,settings,options:{...info.advanced_settings[1],widget_specs:Object.fromEntries(Object.entries(info).map(([k,v])=>[k,v[1]]))},onChange(){},mappingPanel:panel,onSolo(){}});
const host=document.createElement('div');host.style.display='none';host.append(ui.element,panel.element);document.body.append(host);ui.rebuild();
const check=(ok,msg)=>{if(!ok)throw Error(msg)};
const button=(text,root=ui.element)=>[...root.querySelectorAll('button')].find(b=>b.textContent===text);
const input=(name,value)=>{const e=ui.element.querySelector(`input[aria-label="${name}"]`);e.value=value;e.dispatchEvent(new Event('change'));};
const results=[];
try {
[...ui.element.querySelectorAll('button')].find(b=>b.textContent.startsWith('1 ')).click();ui.element.querySelector('[data-target=animation]').click();
check(JSON.parse(settings.value).audio_mappings[0].target==='animation','click assignment');results.push('click assignment');
input('animation mapping 1 End frame (exclusive)','48');
const transfer=new DataTransfer();transfer.setData('application/x-fl-envelope','2');
ui.element.querySelector('[data-target=animation]').dispatchEvent(new DragEvent('drop',{dataTransfer:transfer,bubbles:true,cancelable:true}));button('Add time range').click();
check(JSON.parse(settings.value).audio_mappings[1].start_frame===48,'gap assignment');results.push('drop and non-overlapping range');
input('animation mapping 2 Start frame','47');
check(JSON.parse(settings.value).audio_mappings[1].start_frame===48,'overlap rejection');results.push('overlap rejection');
button('Remove mapping').click();check(JSON.parse(settings.value).audio_mappings.length===1,'remove');
const slider=ui.element.querySelector('input[aria-label="animation slider"]');slider.value='.5';slider.dispatchEvent(new Event('input'));check(node.widgets.find(w=>w.name==='animation').value===.5,'slider');
ui.element.querySelector('[aria-label="Reset animation"]').click();check(node.widgets.find(w=>w.name==='animation').value===.18,'reset');check(JSON.parse(settings.value).audio_mappings.length===1,'reset retained mapping');results.push('slider, reset, removal');
for(const title of ['Camera','Reveals','Overlay','Finish','Layers']){button(title).click();check(ui.element.querySelectorAll('.scan-card').length>0,'tab '+title);}results.push('all tabs');
const spacing=ui.element.querySelector('[data-target=stack_spacing]');
spacing.dispatchEvent(new DragEvent('drop',{dataTransfer:transfer,bubbles:true,cancelable:true}));
check(JSON.parse(settings.value).audio_mappings.some(r=>r.target==='stack_spacing'&&r.source===2),'stack envelope mapping');
const stack=ui.element.querySelector('input[aria-label="stack count"]');stack.value='6';stack.dispatchEvent(new Event('input'));
check(JSON.parse(settings.value).stack_count===6,'stack count');
stack.value='2.5';stack.dispatchEvent(new Event('input'));check(JSON.parse(settings.value).stack_count===6,'integer validation');
button('random on snare').click();check(JSON.parse(settings.value).window_order==='random_on_snare','window order');
check(ui.element.querySelector('[aria-label="Help: window_order"]').title.includes('Envelope 2'),'ordering help');results.push('layer controls and drag mapping');
button('Overlay').click();
const minimum=ui.element.querySelector('input[aria-label="min cut frames"]');minimum.value='40';minimum.dispatchEvent(new Event('input'));
check(JSON.parse(settings.value).max_cut_frames===40,'maximum follows minimum');
check(ui.element.querySelector('input[aria-label="max cut frames"]').value==='40','maximum field synchronized');
const maximum=ui.element.querySelector('input[aria-label="max_cut_frames slider"]');maximum.value='5';maximum.dispatchEvent(new Event('input'));
check(JSON.parse(settings.value).min_cut_frames===5&&minimum.value==='5','minimum follows maximum');results.push('paired cut controls');
return results;
} finally {host.remove();}
}
+31
View File
@@ -35,6 +35,36 @@ def beat_json():
class BeatPromptScheduleTests(unittest.TestCase):
def test_reference_fingerprint_uses_serialized_widget_without_linked_inputs(self):
with mock.patch.object(schedule, "reference_file_fingerprint", return_value=(("image.png", 1, 2),)) as fingerprint:
value = schedule.FL_Audio_Beat_Prompt_Schedule.fingerprint_inputs(reference_schedule="manifest")
fingerprint.assert_called_once_with("manifest")
self.assertEqual(value, (None, (("image.png", 1, 2),)))
def test_frame_schedule_without_detected_beats(self):
payload = json.loads(beat_json())
payload['beat_times'] = []
output = schedule.FL_Audio_Beat_Prompt_Schedule.execute(
beat_positions=json.dumps(payload), timeline='[0 - 60]\nAnime action.',
time_unit='frames', fps=24, sequence_duration=60).result
self.assertEqual(output[1], 60)
self.assertEqual(output[0]['sections'][0]['end'], 2.5)
with self.assertRaisesRegex(ValueError, 'non-empty'):
schedule.FL_Audio_Beat_Prompt_Schedule.execute(
beat_positions=json.dumps(payload), timeline='[0 - 4]\nAnime action.', time_unit='beats')
def test_reference_metadata_survives_schedule_and_frame_payload(self):
metadata = {"version": 1, "assets": {}, "sections": [{"id": "section-a", "mode": "none", "asset_ids": []}]}
output = schedule.FL_Audio_Beat_Prompt_Schedule.execute(
beat_positions=beat_json(), timeline="[0 - 60]\nA runner.", default_fade_in=0,
default_fade_out=0, curve="linear", reference_schedule=json.dumps(metadata),
).result[0]
self.assertEqual(output["sections"][0]["section_id"], "section-a")
self.assertEqual(output["sections"][0]["references"], {"mode": "none", "asset_ids": []})
frames = schedule._frame_sections(output["sections"], 24, 60)
self.assertEqual(frames[0]["section_id"], "section-a")
self.assertEqual(output["reference_assets"], {})
def test_fractional_beats_use_exact_detected_intervals(self):
beats, duration = schedule._load_beats(beat_json())
sections = schedule._resolve_schedule(
@@ -471,6 +501,7 @@ class BeatPromptScheduleTests(unittest.TestCase):
"render_groups",
"analysis_cache_key",
"envelope_layers",
"reference_schedule",
],
)
self.assertEqual(
+44
View File
@@ -0,0 +1,44 @@
import importlib.util
import json
from pathlib import Path
import unittest
import numpy as np
import torch
spec = importlib.util.spec_from_file_location("drum_bands_test", Path(__file__).parents[1] / "nodes/audio/FL_Audio_Drum_Detector.py")
drums = importlib.util.module_from_spec(spec)
spec.loader.exec_module(drums)
class DrumBandTests(unittest.TestCase):
def test_independent_bands_detect_transients_and_keep_duration(self):
rate = 24000
signal = np.zeros(rate * 3, np.float32)
time = np.arange(2400) / rate
for start, frequency in ((0.4, 80), (1.1, 1200), (1.9, 8000)):
burst = np.sin(2 * np.pi * frequency * time) * np.hanning(len(time))
offset = round(start * rate)
signal[offset:offset+len(burst)] += burst.astype(np.float32)
audio = {"waveform": torch.from_numpy(signal).reshape(1, 1, -1), "sample_rate": rate}
result = json.loads(drums.FL_Audio_Drum_Detector().detect_drums(audio, detection_mode="independent_bands")[0])
self.assertEqual(result['duration'], 3)
for key, expected in (("kick_times", .4), ("snare_times", 1.1), ("hihat_times", 1.9)):
self.assertTrue(any(abs(value - expected) < .15 for value in result[key]), (key, result[key]))
self.assertTrue(all(0 <= value < 3 for value in result[key]))
def test_silence_produces_no_fake_hits(self):
audio = {"waveform": torch.zeros(1, 1, 24000), "sample_rate": 24000}
result = json.loads(drums.FL_Audio_Drum_Detector().detect_drums(audio, detection_mode="independent_bands")[0])
for key in ('kick_times', 'snare_times', 'hihat_times'):
self.assertEqual(result[key], [])
def test_existing_workflows_keep_original_default(self):
config = drums.FL_Audio_Drum_Detector.INPUT_TYPES()
self.assertEqual(config['optional']['detection_mode'][1]['default'], 'classified_onsets')
self.assertEqual(list(config['optional'])[:3], ['kick_sensitivity', 'snare_sensitivity', 'hihat_sensitivity'])
if __name__ == '__main__':
unittest.main()
+239
View File
@@ -0,0 +1,239 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
const source = await readFile(new URL("../web/nodes/audio/audio_prompt_references.js", import.meta.url), "utf8");
const helpers = await import(`data:text/javascript;base64,${Buffer.from(source.slice(source.indexOf("export function ensureReferenceIds"), source.indexOf("export function mountReferences"))).toString("base64")}`);
const storySource = await readFile(new URL("../web/nodes/audio/audio_prompt_storyboards.js", import.meta.url), "utf8");
const storyHelpers = await import(`data:text/javascript;base64,${Buffer.from(storySource.slice(storySource.indexOf("const PREFIX"))).toString("base64")}`);
test("thumbnails represent assigned images only, grouping storyboard panels into one sheet", () => {
const assets = { a: { kind: "image", storyboard_id: "job", source: { filename: "sheet.png" } },
b: { kind: "image", storyboard_id: "job" }, unused: { kind: "image" }, audio: { kind: "audio" } };
const clip = { references: { mode: "custom", asset_ids: ["a", "b", "audio"] } };
assert.deepEqual(storyHelpers.timelineReferenceImages(clip, assets), [{key: "job", image: assets.a.source, assetIds: ["a", "b"]}]);
clip.references.mode = "defaults";
assert.deepEqual(storyHelpers.timelineReferenceImages(clip, assets), []);
clip.references = {mode: "custom", asset_ids: []};
assert.deepEqual(storyHelpers.timelineReferenceImages(clip, assets), []);
});
test("reference wiring reuses the schedule library and enables full visual conditioning", () => {
const graph = { _nodes: [], links: {1: {origin_id: 10}, 2: {origin_id: 10}}, change() {} };
const library = {id: 20, type: "FL_Prompt_Reference_Library", inputs: [{name: "prompt_schedule", link: 1}],
connect(output, planner, slot) { graph.links[3] = {origin_id: this.id}; planner.inputs[slot].link = 3; } };
const planner = {type: "FL_MiniMaxH3BeatShotPlanner", inputs: [{name: "prompt_schedule", link: 2}, {name: "reference_library", link: null}], widgets: [{name: "visual_reference_mode", value: "qwen only"}]};
graph._nodes = [library, planner];
const editor = {node: {id: 10, graph}};
storyHelpers.ensureReferenceWiring(editor);
storyHelpers.ensureReferenceWiring(editor);
assert.equal(planner.inputs[1].link, 3);
assert.equal(planner.widgets[0].value, "full");
assert.equal(graph._nodes.length, 2);
assert.throws(() => storyHelpers.ensureReferenceWiring({node: {id: 99, graph}}), /Connect this scheduler/);
});
test("stable IDs survive moves; duplicated clips get new IDs and independent assignments", () => {
const a = { references: { mode: "custom", asset_ids: ["image"] } };
helpers.ensureReferenceIds([a]);
const id = a.sectionId;
const duplicate = { ...a };
helpers.ensureReferenceIds([a, duplicate]);
assert.equal(a.sectionId, id);
assert.notEqual(duplicate.sectionId, id);
duplicate.references.asset_ids.push("other");
assert.deepEqual(a.references.asset_ids, ["image"]);
helpers.ensureReferenceIds([duplicate, a]);
assert.equal(a.sectionId, id);
});
test("reference document survives serialization and reload", () => {
const editor = { clips: [{ prompt: "A" }, { prompt: "B", references: { mode: "none", asset_ids: [] } }], referenceAssets: {}, widgets: { referenceSchedule: {} } };
helpers.serializeReferences(editor);
const ids = editor.clips.map(clip => clip.sectionId);
const restored = { clips: [{}, {}], widgets: editor.widgets };
helpers.loadReferences(restored);
assert.deepEqual(restored.clips.map(clip => clip.sectionId), ids);
assert.equal(restored.clips[1].references.mode, "none");
restored.clips.pop();
assert.throws(() => helpers.loadReferences(restored), /no longer match/);
});
test("Partner submission uses scoped credentials without changing the shared API", async () => {
const source = await readFile(new URL("../web/nodes/audio/audio_prompt_storyboards.js", import.meta.url), "utf8");
const body = source.slice(source.indexOf("const PREFIX"));
const bindings = `
const api = { clientId: 'client', fetchApi() {} };
const app = { extensionManager: { _p: { _s: new Map([
['auth', {getAuthToken: async () => 'fixture-token'}],
['apiKeyAuth', {getApiKey: () => undefined}]
]) } } };
class ComfyApi { async queuePrompt(index, prompt) {
return { authenticated: !!this.authToken, sharedUntouched: !api.authToken,
isolated: this !== api, index, prompt };
} }
`;
const module = await import(`data:text/javascript;base64,${Buffer.from(bindings + body).toString("base64")}`);
const submit = await module.prepareStoryboardQueue();
const result = await submit({ output: "fixture" }, "job");
assert.equal(result.authenticated, true);
assert.equal(result.sharedUntouched, true);
assert.equal(result.isolated, true);
assert.equal(result.prompt.workflow.extra.storyboard_id, "job");
assert.equal(JSON.stringify(result.prompt).includes("fixture-token"), false);
});
test("Missing Partner sign-in stops before any queue call", async () => {
const source = await readFile(new URL("../web/nodes/audio/audio_prompt_storyboards.js", import.meta.url), "utf8");
const bindings = `
const api = {};
const app = { extensionManager: { _p: { _s: new Map([
['auth', {getAuthToken: async () => null}], ['apiKeyAuth', {getApiKey: () => null}]
]) } } };
class ComfyApi { queuePrompt() { throw new Error('Must not queue'); } }
`;
const module = await import(`data:text/javascript;base64,${Buffer.from(bindings + source.slice(source.indexOf("const PREFIX"))).toString("base64")}`);
await assert.rejects(module.prepareStoryboardQueue(), /Sign in/);
});
test("Writer generation queues directly, attaches completed panels, and rerolls without clearing active references", async () => {
class Element {
children = [];
dataset = {};
style = {};
selectors = new Map();
append(...items) { this.children.push(...items); }
after() {}
remove() {}
setAttribute(key,value) { this[key]=value; }
replaceChildren(...items) { this.children = items; }
querySelector(selector) {
if (!this.selectors.has(selector)) this.selectors.set(selector, new Element());
return this.selectors.get(selector);
}
}
let poll;
let queued = 0;
let extracted = 0;
const jobs = [];
const parent = new Element();
const clip = {sectionId: "section", start: 0, end: 121, prompt: "Fast motion", references: {mode: "defaults", asset_ids: []}};
const graph = {links: {1: {origin_id: 10}, 2: {origin_id: 10}, 3: {origin_id: 20}}, change() {}};
graph._nodes = [
{id: 20, type: "FL_Prompt_Reference_Library", inputs: [{name: "prompt_schedule", link: 1}]},
{type: "FL_MiniMaxH3BeatShotPlanner", inputs: [{name: "prompt_schedule", link: 2}, {name: "reference_library", link: 3}], widgets: []},
];
const editor = {node: {id: 10, graph}, clips: [clip], referenceAssets: {}, canvas: {parentElement: parent},
clipRects: [{index: 0, x: 10, y: 20, width: 250, height: 200}],
runEdit(label, fn) { fn(); }, serialize() {}, syncInspector() {},
scheduleDraw() { this.renderStoryboardThumbnails?.(); }};
const errors = [];
const writer = {editor, nodeSettings: {schedulerId: "schedule", moodboards: [null, null, null, null]},
root: new Element(), saveNodeSettings() {}, showError(message) { errors.push(message); },
client: {imageUrl(image) { return image.filename; }},
currentDocument: {allowed_indices: [0], boxes: [{index: 0, start_frame: 0, end_frame: 121, prompt: clip.prompt}],
sectionIds: {0: "section"}, referenceSelections: {0: structuredClone(clip.references)}}};
const api = {clientId: "fixture", apiURL:path=>path, async fetchApi(path, options) {
const body = options?.body ? JSON.parse(options.body) : null;
let result;
if (path.includes("?scheduler_id=")) result = jobs;
else if (path.endsWith("/storyboards")) {
result = jobs.find(job => job.spec.request_key === body.request_key);
if (!result) { result = {id: String(jobs.length + 1), state: "proposed", spec: body, result: {}}; jobs.unshift(result); }
} else {
const [, id, action] = path.match(/\/([^/]+)\/([^/]+)$/);
const job = jobs.find(job => job.id === id);
if (id === "batch" && action === "submit") { result={graph:{}};for(const id of body.job_ids){jobs.find(j=>j.id===id).state="submitted";result.graph[id]={class_type:"GeminiNanoBanana2V2"};} }
else if (action === "submit") { job.state = "submitted"; result = {graph: {storyboard: {class_type: "GeminiNanoBanana2V2"}}}; }
else if (action === "receipt") { job.result.prompt_id = body.prompt_id; result = job; }
else if (action === "refresh") result = job;
else if (action === "extract") {
extracted++;
job.result.assets = Object.fromEntries(Array.from({length: 4}, (_, index) => [`${id}-${index}`, {
kind: "image", filename: `panel-${id}-${index}.png`, storyboard_id: id, source: job.result.source,
}]));
result = job;
} else throw Error(`Unexpected action ${action}`);
}
return {ok: true, json: async () => structuredClone(result)};
}};
globalThis.__storyboardTest = {
api, app: {extensionManager: {_p: {_s: new Map([
["auth", {getAuthToken: async () => "fixture"}], ["apiKeyAuth", {getApiKey: () => null}],
])}}},
ComfyApi: class { async queuePrompt() { queued++; return {prompt_id: `queue-${queued}`}; } },
document: {createElement: () => new Element(), createTextNode: value => value},
setTimeout: callback => { poll = callback; return 1; }, clearTimeout() {},
};
const bindings = "const {api, app, ComfyApi, document, setTimeout, clearTimeout} = globalThis.__storyboardTest;\n";
const module = await import(`data:text/javascript;base64,${Buffer.from(bindings + storySource.slice(storySource.indexOf("const PREFIX"))).toString("base64")}`);
const flush = async () => { for (let i = 0; i < 8; i++) await new Promise(resolve => setImmediate(resolve)); };
const descendants = root => (root.children||[]).flatMap(child=>[child,...descendants(child)]);
try {
module.mountStoryboards(writer);
await flush();
await writer.generateStoryboards([{index: 0, grid: 2, prompt: "Four running poses"}], [], "message");
assert.equal(queued, 1);
assert.equal(clip.references.mode, "defaults");
jobs[0].state = "complete";
jobs[0].result.source = {filename: "sheet.png"};
await poll();
assert.equal(extracted, 1);
assert.equal(clip.references.asset_ids.length, 4);
assert.equal(module.timelineReferenceImages(clip, editor.referenceAssets).length, 1);
const original = structuredClone(clip.references);
const row = parent.children[0].children[0];
await descendants(row).find(child => child.title?.startsWith("Reroll")).onclick();
assert.equal(queued, 2);
assert.deepEqual(clip.references, original);
jobs[0].state = "complete";
jobs[0].result.source = {filename: "reroll.png"};
await poll();
assert.notDeepEqual(clip.references, original);
assert.equal(module.timelineReferenceImages(clip, editor.referenceAssets)[0].image.filename, "reroll.png");
descendants(parent.children[0].children[0]).find(child => child.title?.startsWith("Remove")).onclick();
await poll();
assert.deepEqual(clip.references.asset_ids, []);
assert.equal(queued, 2);
writer.currentDocument.referenceSelections[0] = structuredClone(clip.references);
await writer.generateStoryboards([{index: 0, grid: 2, prompt: "New storyboard"}], [], "next-message");
assert.equal(queued, 3);
clip.prompt = "User edited the section during generation";
jobs[0].state = "complete";
jobs[0].result.source = {filename: "stale.png"};
await poll();
assert.deepEqual(clip.references.asset_ids, []);
assert.equal(extracted, 2);
assert.equal(writer.nodeSettings.storyboardResults.includes(jobs[0].id), false, "edited sections must not consume saved results");
const savedJob = jobs[0];
writer.disposeStoryboards();
parent.replaceChildren();
module.mountStoryboards(writer);
await flush();
await poll();
const attach = descendants(parent).find(child => child.textContent === "Attach");
assert.ok(attach, "unattached saved images remain recoverable after reopening");
await attach.onclick();
assert.equal(extracted, 3);
assert.equal(queued, 3, "recovering a saved image never queues a paid generation");
assert.equal(clip.prompt, "User edited the section during generation");
assert.equal(clip.references.asset_ids.length, 4);
assert.ok(writer.nodeSettings.storyboardResults.includes(savedJob.id));
await poll();
assert.equal(extracted, 3, "attached results are not repeatedly extracted");
editor.clips=Array.from({length:4},(_,i)=>({sectionId:`batch-${i}`,start:i*24,end:(i+1)*24,prompt:`Scene ${i}`,references:{mode:"defaults",asset_ids:[]}}));
writer.currentDocument={allowed_indices:[0,1,2,3],boxes:editor.clips.map((c,index)=>({index,start_frame:c.start,end_frame:c.end,prompt:c.prompt})),sectionIds:Object.fromEntries(editor.clips.map((c,i)=>[i,c.sectionId])),referenceSelections:Object.fromEntries(editor.clips.map((c,i)=>[i,structuredClone(c.references)]))};
await writer.generateStoryboards(editor.clips.map((c,index)=>({index,grid:2,prompt:c.prompt})),[],"batch-message");
assert.equal(queued,4,"all four images use one independent async graph");
const batchJobs=jobs.filter(j=>j.spec.section_id.startsWith('batch-'));
assert.equal(batchJobs.length,4);
assert.equal(new Set(batchJobs.map(j=>j.spec.continuity)).size,1);
assert.match(batchJobs[0].spec.continuity,/Scene 0/);
assert.match(batchJobs[0].spec.continuity,/Scene 3/);
assert.deepEqual(errors, []);
} finally {
writer.disposeStoryboards();
delete globalThis.__storyboardTest;
}
});
+34 -1
View File
@@ -8,7 +8,9 @@ async function importModuleBody(filename, startMarker) {
const source = await readFile(new URL(filename, AUDIO_NODE_URL), "utf8");
const start = source.indexOf(startMarker);
assert.notEqual(start, -1);
const encoded = Buffer.from(source.slice(start)).toString("base64");
const references = await readFile(new URL("audio_prompt_references.js", AUDIO_NODE_URL), "utf8");
const referenceHelpers = filename === "audio_prompt_sequencer_editor.js" ? references.slice(references.indexOf("export function ensureReferenceIds"), references.indexOf("export function mountReferences")) : "";
const encoded = Buffer.from(referenceHelpers + source.slice(start)).toString("base64");
return import(`data:text/javascript;base64,${encoded}`);
}
@@ -60,6 +62,37 @@ test("playhead draws reuse the static timeline layer", async () => {
assert.equal(playheadDraws, 2);
});
test("playback redraws update envelope indicators without rebuilding previews", async () => {
const module = await importModuleBody("audio_prompt_sequencer_editor.js", "const EPSILON");
const editor = Object.create(module.BeatPromptSequencer.prototype);
let callback;
let previewSyncs = 0;
const positions = [];
editor.pendingFrame = null;
editor.staticDirty = false;
editor.draw = () => { editor.staticDirty = false; };
editor.syncEnvelopePreviews = () => { previewSyncs++; editor.updateEnvelopePlayheads(); };
editor.updateEnvelopePlayheads = () => positions.push(editor.playheadFrame);
const previousRAF = globalThis.requestAnimationFrame;
globalThis.requestAnimationFrame = fn => { callback = fn; return 1; };
try {
for (const frame of [12, 24, 0]) {
editor.playheadFrame = frame;
editor.scheduleDraw(false);
callback();
}
assert.deepEqual(positions, [12, 24, 0]);
assert.equal(previewSyncs, 0);
editor.scheduleDraw(false);
editor.scheduleDraw(true);
callback();
assert.equal(previewSyncs, 1);
assert.deepEqual(positions, [12, 24, 0, 0]);
} finally {
globalThis.requestAnimationFrame = previousRAF;
}
});
test("Writer activity animates one active box and clears completed boxes independently", async () => {
const module = await importModuleBody("audio_prompt_sequencer_editor.js", "const EPSILON");
const editor = Object.create(module.BeatPromptSequencer.prototype);
+101
View File
@@ -4,6 +4,105 @@ import test from "node:test";
const AUDIO_NODE_URL = new URL("../web/nodes/audio/", import.meta.url);
test("generated storyboards take precedence over simultaneous reference assignments", async () => {
const source = await readFile(new URL("audio_prompt_writer.js", AUDIO_NODE_URL), "utf8");
const module = await import(`data:text/javascript;base64,${Buffer.from(source.slice(source.indexOf("const NODE_DEFAULTS"))).toString("base64")}`);
const writer = Object.create(module.BeatPromptWriter.prototype);
const calls = [];
Object.assign(writer, {
runProgress: {}, writerActivity: {phase:"complete"}, statusElement:{dataset:{state:"applied"}},
generateStoryboards(actions) { calls.push(["generate", actions]); },
applyReferenceAssignments(actions) { calls.push(["assign", actions]); },
finishAssistantMessage() {}, scrollToBottom() {},
});
const storyboards = [{index:0, prompt:"Story"}];
writer.handleRunEvent({type:"run_finished", assistantMessage:{id:"message", metadata:{storyboards,
reference_assignments:[{index:0,mode:"none",asset_ids:[]},{index:1,mode:"custom",asset_ids:["existing"]}]}}});
assert.deepEqual(calls, [["generate", storyboards], ["assign", [{index:1,mode:"custom",asset_ids:["existing"]}]]]);
});
async function modelPicker(t) {
const source = await readFile(new URL("audio_prompt_writer.js", AUDIO_NODE_URL), "utf8");
assert.match(source, /<select data-writer-setting="model-select"/);
assert.match(source, /this\.modelInput\.value = this\.modelSelect\.value/);
assert.match(source, /this\.discoverModels\(false\)/);
const module = await import(`data:text/javascript;base64,${Buffer.from(source.slice(source.indexOf("const NODE_DEFAULTS"))).toString("base64")}`);
const originalDocument = globalThis.document;
globalThis.document = { createElement: () => ({}) };
t.after(() => { globalThis.document = originalDocument; });
const select = () => ({ value: "", children: [], replaceChildren(...children) { this.children = children; } });
const writer = Object.create(module.BeatPromptWriter.prototype);
Object.assign(writer, {
settings: { provider: "codex_subscription", presets: { codex_subscription: { reasoning_efforts: ["low", "high", "ultra"] } } },
providerSelect: { value: "codex_subscription" }, modelInput: { value: "saved-model" },
modelSelect: select(), modelOptions: select(), modelStatus: {}, models: [], modelRequest: 0,
reasoningSelect: { ...select(), value: "ultra" }, reasoningComposer: select(), toast() {},
});
return writer;
}
test("subscription dropdown loads all models, retains selection, and constrains reasoning", async (t) => {
const writer = await modelPicker(t);
writer.client = { models: async (refresh) => {
assert.equal(refresh, false);
return { models: [{ id: "saved-model", label: "Saved", reasoningEfforts: ["low", "high"] }, { id: "another-model", label: "Another" }] };
} };
await writer.discoverModels(false);
assert.deepEqual(writer.modelSelect.children.map((item) => item.value), ["saved-model", "another-model"]);
assert.equal(writer.modelSelect.value, "saved-model");
assert.equal(writer.reasoningSelect.value, "default");
assert.match(writer.modelStatus.textContent, /2 models available/);
});
test("empty or failed model discovery does not clear the saved model", async (t) => {
const writer = await modelPicker(t);
writer.client = { models: async () => ({ models: [] }) };
await writer.discoverModels();
assert.equal(writer.modelSelect.value, "saved-model");
assert.match(writer.modelSelect.children[0].textContent, /not in loaded list/);
assert.match(writer.modelStatus.textContent, /No models returned/);
writer.client.models = async () => { throw new Error("Unavailable"); };
await writer.discoverModels();
assert.equal(writer.modelSelect.value, "saved-model");
assert.match(writer.modelStatus.textContent, /Could not load models: Unavailable/);
});
test("model discovery ignores stale responses and unsaved provider switches", async (t) => {
const writer = await modelPicker(t);
let resolveFirst;
writer.client = { models: () => new Promise((resolve) => { resolveFirst = resolve; }) };
const pending = writer.discoverModels();
writer.client.models = async () => ({ models: [{ id: "new-model" }] });
await writer.discoverModels();
resolveFirst({ models: [{ id: "old-model" }] });
await pending;
assert.equal(writer.models[0].id, "new-model");
writer.providerSelect.value = "claude_subscription";
writer.client.models = () => { throw new Error("Must not query the saved Codex provider"); };
await writer.discoverModels();
assert.match(writer.modelStatus.textContent, /Save the provider connection/);
});
test("saving the dropdown choice forwards its model id", async (t) => {
const writer = await modelPicker(t);
writer.modelSelect.value = "another-model";
writer.modelInput.value = writer.modelSelect.value;
writer.credentialInput = { value: "" };
writer.baseUrlInput = { value: "" };
writer.temperatureInput = { value: "0.4" };
writer.maxTokensInput = { value: "16384" };
let saved;
writer.client = {
updateSettings: async (settings) => { saved = settings; return settings; },
status: async () => ({ configured: true }),
};
writer.populateSettings = writer.updateProviderBadge = writer.setStatus = () => {};
writer.discoverModels = async () => {};
await writer.saveSettings();
assert.equal(saved.model, "another-model");
assert.equal(saved.provider, "codex_subscription");
});
async function loadClientModule(fetchApi) {
const source = await readFile(new URL("audio_prompt_writer_client.js", AUDIO_NODE_URL), "utf8");
const start = source.indexOf("const ROOT");
@@ -21,6 +120,8 @@ test("standalone writer panel remains a valid ESM module", async () => {
const encoded = Buffer.from(source.slice(start)).toString("base64");
const module = await import(`data:text/javascript;base64,${encoded}`);
assert.equal(typeof module.BeatPromptWriter, "function");
assert.doesNotMatch(source, /STARTERS|welcomeElement|flbps-writer-welcome|flbps-writer-starters|action === "starter"/);
assert.match(source, /data-writer-role="thread"/);
assert.doesNotMatch(source, /FL_MCP|\/api\/chat|MCPServer/);
assert.match(source, /client\.startRun/);
assert.match(source, /event\.revision !== this\.currentDocument\.revision/);
+45
View File
@@ -0,0 +1,45 @@
import importlib.util
from pathlib import Path
import unittest
from unittest.mock import patch
import torch
spec = importlib.util.spec_from_file_location("separation_test", Path(__file__).parents[1] / "nodes/audio/FL_Audio_Separation.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
class IdentitySeparator(torch.nn.Module):
sources = ("drums", "bass", "other", "vocals")
def forward(self, mix):
return mix.unsqueeze(1).expand(-1, 4, -1, -1)
class SeparationChunkTests(unittest.TestCase):
def test_loading_failure_does_not_return_fake_stems(self):
with patch("torchaudio.pipelines.HDEMUCS_HIGH_MUSDB_PLUS.get_model", side_effect=RuntimeError("missing weights")):
with self.assertRaisesRegex(RuntimeError, "FL Audio Separation failed: missing weights"):
module.FL_Audio_Separation().separate_audio({"waveform": torch.ones(1, 2, 100), "sample_rate": 44100})
def test_overlaps_and_tail_reconstruct_without_gaps(self):
node = module.FL_Audio_Separation()
for length in (3, 20, 21, 57, 101):
for overlap in (0, .2, .7):
for fade in ("linear", "half_sine", "logarithmic", "exponential"):
with self.subTest(length=length, overlap=overlap, fade=fade):
mix = torch.rand(1, 2, length)
result = node._separate_sources(IdentitySeparator(), mix, 10,
segment=2, overlap=overlap, chunk_fade_shape=fade)
torch.testing.assert_close(result, mix.unsqueeze(1).expand(-1, 4, -1, -1))
def test_invalid_overlap(self):
with self.assertRaisesRegex(ValueError, "shorter"):
module.FL_Audio_Separation()._separate_sources(IdentitySeparator(), torch.ones(1, 2, 30),
10, segment=1, overlap=1)
if __name__ == "__main__":
unittest.main()
+197
View File
@@ -0,0 +1,197 @@
import importlib
import json
from pathlib import Path
import sys
import tempfile
from types import ModuleType
import unittest
from unittest.mock import patch
import av
import torch
root = Path(__file__).parents[1] / "nodes"
for name, path in (("interactive_test",root),("interactive_test.vfx",root/"vfx"),("interactive_test.audio",root/"audio")):
package=ModuleType(name)
package.__path__=[str(path)]
sys.modules[name]=package
fx=importlib.import_module("interactive_test.vfx.FL_InteractiveScanFX")
class InteractiveScanTests(unittest.TestCase):
def test_layer_controls_and_mappings_render_deterministically(self):
args = self.inputs()
settings = fx.DEFAULTS | {"stack_count":6,"stack_palette":"cyan","stack_rotation":4,
"window_order":"random_on_snare","window_blend":"screen","window_fade_in":.1,"window_fade_out":.1,
"depth_weight":50,"depth_opacity":.6,"audio_mappings":[
{"source":0,"target":"stack_spacing","minimum":.5,"maximum":3},
{"source":1,"target":"stack_rotation","minimum":-5,"maximum":5},
{"source":2,"target":"voxel_opacity","minimum":.2,"maximum":1}]}
args["advanced_settings"] = json.dumps(settings)
with patch.object(fx,"write_preview",return_value={}):
first = fx.FL_InteractiveScanFX.execute(**args)
second = fx.FL_InteractiveScanFX.execute(**args)
args["advanced_settings"] = "{}"
default = fx.FL_InteractiveScanFX.execute(**args)
torch.testing.assert_close(first.args[0],second.args[0],rtol=0,atol=0)
self.assertFalse(torch.equal(first.args[0],default.args[0]))
self.assertTrue(torch.isfinite(first.args[0]).all())
for invalid in ({"stack_count":2.5},{"stack_count":9},{"window_blend":"unknown"},{"window_fade_in":-1}):
with self.assertRaises(ValueError):fx.settings_from_json(json.dumps(invalid))
def test_dynamic_shots_accept_any_section_count(self):
images = torch.rand(61, 8, 8, 3)
for count in (1, 2, 4, 8, 13):
schedule = {"sections": [{"start_frame": 61*i//count, "end_frame": 61*(i+1)//count} for i in range(count)]}
chunks, = fx.FL_ScanVideoShots().split(images, schedule)
self.assertEqual(len(chunks), count)
torch.testing.assert_close(torch.cat(chunks), images)
def test_dynamic_shot_list_preserves_groups_and_remainders(self):
images = torch.rand(17, 8, 8, 3)
schedule = {"sections": [
{"start_frame": 0, "end_frame": 3},
{"start_frame": 3, "end_frame": 6, "render_group": 1},
{"start_frame": 6, "end_frame": 10, "render_group": 1},
{"start_frame": 10, "end_frame": 17},
]}
chunks, = fx.FL_ScanVideoShots().split(images, schedule)
self.assertEqual([len(c) for c in chunks], [3, 7, 7])
torch.testing.assert_close(torch.cat(chunks), images)
self.assertNotEqual(chunks[0].data_ptr(), images.data_ptr())
with self.assertRaisesRegex(ValueError, "length must match"):
fx.FL_ScanVideoShots().split(torch.rand(18,8,8,3), schedule)
def test_collected_analysis_matches_manual_connections(self):
args = self.inputs()
shot = args["analysis"]["shot0"]
with patch.object(fx, "write_preview", return_value={}):
expected = fx.FL_InteractiveScanFX.execute(**args)
bundle, = fx.FL_ScanAnalysisCollect().collect([shot])
args["analysis"] = {"shot0": bundle}
actual = fx.FL_InteractiveScanFX.execute(**args)
torch.testing.assert_close(actual.args[0], expected.args[0])
def test_finishing_falls_back_to_cpu_with_low_gpu_headroom(self):
args = self.inputs()
with patch.object(fx, "write_preview", return_value={}), patch.object(fx.model_management, "get_torch_device", return_value=torch.device("cpu")):
expected = fx.FL_InteractiveScanFX.execute(**args)
with patch.object(fx, "write_preview", return_value={}), patch.object(fx.model_management, "get_torch_device", return_value=torch.device("cuda")), patch.object(fx.model_management, "get_free_memory", return_value=0):
actual = fx.FL_InteractiveScanFX.execute(**args)
torch.testing.assert_close(actual.args[0], expected.args[0], rtol=0, atol=0)
def inputs(self):
torch.manual_seed(5)
images=torch.rand(12,32,32,3)
depth=torch.rand_like(images)
normals=torch.rand_like(images)
shot,=fx.FL_ScanAnalysis().pack(depth,normals)
envelope={"type":"fl_audio_envelope","version":1,"fps":24,"duration":.5,"total_frames":12,"values":[float(i%4==0) for i in range(12)]}
return dict(images=images,analysis={"shot0":shot},kick_envelope=envelope,snare_envelope=envelope,hihat_envelope=envelope,
fps=24,cube_size=8,relief=.65,animation=.18,speed=.7,cursor_count=2,motion_strength=.6,seed=73,advanced_settings=json.dumps(fx.DEFAULTS))
def test_matches_existing_effect_chain(self):
args=self.inputs();s=fx.DEFAULTS;shot=args["analysis"]["shot0"];im=args["images"];original=im.clone();e=args["kick_envelope"]
voxels,=fx.FL_VoxelNormalRelief().render(shot["normals"],shot["depth"],8,.65,.18,.7,24,41)
tracks={"frames":[[] for _ in im],"height":32,"width":32}
scan,_,surface=fx.FL_StreetScanComposite().render(im,shot["depth"],voxels,torch.zeros(12,32,32),tracks,24,41,5.5,1,.74,0,0,.7,0,None,.5,"digital_layers")
expected,_,mask,report=fx.FL_ScanAudioEdit().render(im,scan,surface,e,e,e,"12",24,73,10,20,1.7,2,1.2,1,.6,"audio_locked")
expected,=fx.FL_Audio_Reactive_Brightness().apply_brightness(expected,e,mask=mask[:,:,:,None].expand(-1,-1,-1,3),brightness_intensity=.16)
expected,=fx.FL_Audio_Reactive_Saturation().apply_saturation(expected,e,base_saturation=.9,saturation_intensity=.3)
expected,=fx.FL_Audio_Reactive_Edge_Glow().apply_edge_glow(expected,e,edge_threshold=.15,glow_intensity=0,envelope_intensity=.28,glow_color="white",blend_mode="screen")
with patch.object(fx,"write_preview",return_value={}): result=fx.FL_InteractiveScanFX.execute(**args)
torch.testing.assert_close(result.args[0],expected)
torch.testing.assert_close(result.args[1],surface)
torch.testing.assert_close(result.args[2],mask)
torch.testing.assert_close(im,original)
self.assertEqual(result.args[3],report)
def test_preview_encodes_every_frame(self):
args=self.inputs()
with tempfile.TemporaryDirectory() as directory,patch.object(fx.folder_paths,"get_temp_directory",return_value=directory),patch.object(fx,"scan_progress") as progress:
result=fx.FL_InteractiveScanFX.execute(**args)
preview=result.ui["fl_interactive_scan"][0]
with av.open(str(Path(directory)/preview["filename"])) as video:
frames=list(video.decode(video=0))
self.assertEqual(len(frames),12)
self.assertEqual((frames[0].width,frames[0].height),(96,64))
self.assertEqual(preview["envelopes"][0],args["kick_envelope"]["values"])
stages=[call.args[0] for call in progress.call_args_list]
self.assertEqual(list(dict.fromkeys(stages)),["Shot 1/1 · Voxel normals","Shot 1/1 · Depth projection",
"Cursor reveals and audio edit","Color and glow","Encoding previews","Complete"])
self.assertEqual(progress.call_args.args,("Complete",1,1,False))
def test_invalid_timing_rejected_before_render(self):
args=self.inputs();args["fps"]=30
with self.assertRaisesRegex(ValueError,"FPS"):fx.FL_InteractiveScanFX.execute(**args)
def test_analysis_lengths_rejected(self):
args=self.inputs();args["images"]=args["images"][:8]
with self.assertRaisesRegex(ValueError,"video has 8 frames, but analysis covers 12"):fx.FL_InteractiveScanFX.execute(**args)
def test_dynamic_sections_cover_video_without_gaps_or_duplicates(self):
for count in (4, 12, 192, 384, 385, 391):
images=torch.arange(count).reshape(count,1,1,1)
chunks=[fx.FL_ScanVideoSection().split(images,4,i)[0] for i in range(4)]
torch.testing.assert_close(torch.cat(chunks),images)
self.assertLessEqual(max(map(len,chunks))-min(map(len,chunks)),1)
self.assertTrue(all(c.untyped_storage().data_ptr()!=images.untyped_storage().data_ptr() for c in chunks))
def test_dynamic_sections_render_as_aligned_analysis(self):
args=self.inputs();shot=args['analysis']['shot0'];args['analysis']={}
for i in range(4):
depth,=fx.FL_ScanVideoSection().split(shot['depth'],4,i)
normals,=fx.FL_ScanVideoSection().split(shot['normals'],4,i)
args['analysis'][f'shot{i}']=fx.FL_ScanAnalysis().pack(depth,normals)[0]
with patch.object(fx,'write_preview',return_value={}):
result=fx.FL_InteractiveScanFX.execute(**args)
self.assertEqual(len(result.args[0]),len(args['images']))
def test_settings_validation(self):
for bad in ({"oops":1},{"scene_scale":-1},{"edge_threshold":1},{"surface_seed":.5},{"glow_color":"nope"}):
with self.assertRaises(ValueError):fx.settings_from_json(json.dumps(bad))
def test_reversed_cut_range_is_normalized_and_renders(self):
args=self.inputs();args['advanced_settings']=json.dumps({'min_cut_frames':20,'max_cut_frames':3})
settings=fx.settings_from_json(args['advanced_settings'])
self.assertEqual((settings['min_cut_frames'],settings['max_cut_frames']),(3,20))
with patch.object(fx,'write_preview',return_value={}):
result=fx.FL_InteractiveScanFX.execute(**args)
self.assertEqual(len(result.args[0]),12)
def test_multiple_shots_preserve_authored_boundaries(self):
args=self.inputs();shot=args["analysis"]["shot0"]
a,=fx.FL_ScanAnalysis().pack(shot["depth"][:6],shot["normals"][:6])
b,=fx.FL_ScanAnalysis().pack(shot["depth"][6:],shot["normals"][6:])
args["analysis"]={"shot1":b,"shot0":a}
with patch.object(fx,"write_preview",return_value={}):result=fx.FL_InteractiveScanFX.execute(**args)
report=json.loads(result.args[3])
self.assertEqual(report["source_indices"],list(range(12)))
self.assertTrue(any(s["start_frame"]==6 and s["shot"]==2 for s in report["segments"]))
def test_reveals_only_preserves_scene_without_cursors(self):
args=self.inputs();args["cursor_count"]=0
with patch.object(fx,"write_preview",return_value={}):
original=fx.FL_InteractiveScanFX.execute(**args)
args["advanced_settings"]=json.dumps(fx.DEFAULTS|{"motion_mode":"depth_parallax","parallax_scope":"reveals_only","offset_x":.15,"dolly":.2})
reveal=fx.FL_InteractiveScanFX.execute(**args)
args["advanced_settings"]=json.dumps(fx.DEFAULTS|{"motion_mode":"depth_parallax","offset_x":.15,"dolly":.2})
whole=fx.FL_InteractiveScanFX.execute(**args)
torch.testing.assert_close(original.args[0],reveal.args[0])
self.assertFalse(torch.equal(original.args[1],reveal.args[1]))
self.assertFalse(torch.equal(original.args[0],whole.args[0]))
def test_depth_windows_and_mappings_execute(self):
args=self.inputs()
args["advanced_settings"]=json.dumps(fx.DEFAULTS|{"motion_mode":"depth_parallax","depth_style":"contours",
"voxel_weight":0,"edge_weight":0,"depth_weight":100,
"audio_mappings":[{"source":0,"target":"dolly","minimum":0,"maximum":.1,"start_frame":0,"end_frame":12}]})
with patch.object(fx,"write_preview",return_value={}) as preview:result=fx.FL_InteractiveScanFX.execute(**args)
report=json.loads(result.args[3])
self.assertTrue(report["cursor_events"])
self.assertTrue(all(e["effect"]==2 for e in report["cursor_events"]))
self.assertTrue(result.args[2].any())
self.assertEqual(preview.call_args.args[6].shape,(12,32,32))
if __name__=="__main__":unittest.main()
+90
View File
@@ -0,0 +1,90 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
const mappingSource=await readFile(new URL("../web/nodes/vfx/scan_audio_mapping.js",import.meta.url),"utf8");
const mappingURL=`data:text/javascript;base64,${Buffer.from(mappingSource).toString("base64")}`;
const previsSource=await readFile(new URL("../web/nodes/vfx/scan_previs.js",import.meta.url),"utf8");
const previsURL=`data:text/javascript;base64,${Buffer.from(previsSource).toString("base64")}`;
const controlsSource=(await readFile(new URL("../web/nodes/vfx/scan_controls.js",import.meta.url),"utf8")).replace('"./scan_previs.js"',JSON.stringify(previsURL));
const controlsURL=`data:text/javascript;base64,${Buffer.from(controlsSource).toString("base64")}`;
const source = (await readFile(new URL("../web/nodes/vfx/FL_InteractiveScanFX.js", import.meta.url), "utf8"))
.replace('"./scan_previs.js"',JSON.stringify(previsURL))
.replace('"./scan_controls.js"',JSON.stringify(controlsURL))
.replace('"./scan_audio_mapping.js"',JSON.stringify(mappingURL))
.replace('import { app } from "../../../../scripts/app.js";', 'const app = {registerExtension() {}};')
.replace('import { api } from "../../../../scripts/api.js";', 'const api = {};');
const { previewFrame, frameSeekTime } = await import(`data:text/javascript;base64,${Buffer.from(source).toString("base64")}`);
test("preview meters use the displayed frame and clamp the end", () => {
assert.equal(previewFrame(0,24,192),0);
assert.equal(previewFrame(1.5,24,192),36);
assert.equal(previewFrame(8,24,192),191);
assert.equal(previewFrame(-1,24,192),0);
assert.equal(previewFrame(0.999999999/24,24,192),1);
});
const {mappingRange,moveMappingRange}=await import(mappingURL);
const {demoPoint,EFFECT_GROUPS}=await import(previsURL);
const {createPrevis}=await import(previsURL);
test('demo stops scheduling when paused, offscreen, hidden, or disposed', () => {
const original = Object.fromEntries(['document','IntersectionObserver','requestAnimationFrame','cancelAnimationFrame'].map(k=>[k,globalThis[k]]));
let visible, visibility;
const callbacks = new Map();let next = 0;
const element = () => ({style:{}, children:[], append(...items){this.children.push(...items);}});
globalThis.document = {hidden:false,createElement:element,
addEventListener(name,fn){visibility=fn;},removeEventListener(){visibility=null;}};
globalThis.IntersectionObserver = class {constructor(fn){visible=fn;}observe(){}disconnect(){visible=null;}};
globalThis.requestAnimationFrame = fn => {callbacks.set(++next,fn);return next;};
globalThis.cancelAnimationFrame = id => callbacks.delete(id);
try {
const demo=createPrevis(()=>({}));
assert.equal(callbacks.size,0);
visible([{isIntersecting:true}]);assert.equal(callbacks.size,1);
demo.element.children[2].onclick();assert.equal(callbacks.size,0);
demo.element.children[2].onclick();assert.equal(callbacks.size,1);
visible([{isIntersecting:false}]);assert.equal(callbacks.size,0);
visible([{isIntersecting:true}]);assert.equal(callbacks.size,1);
document.hidden=true;visibility();assert.equal(callbacks.size,0);
document.hidden=false;visibility();assert.equal(callbacks.size,1);
demo.setActive(false);assert.equal(callbacks.size,0);
demo.setActive(true);assert.equal(callbacks.size,1);
demo.dispose();assert.equal(callbacks.size,0);assert.equal(visibility,null);
} finally {for(const [key,value] of Object.entries(original)){if(value===undefined)delete globalThis[key];else globalThis[key]=value;}}
});
const {mappingConflict,mappingTarget,normalizeCutRange}=await import(controlsURL);
test('cut bounds follow the edited control and repair reversed saved ranges',()=>{
assert.deepEqual(normalizeCutRange({min_cut_frames:30,max_cut_frames:10},'min_cut_frames'),{min_cut_frames:30,max_cut_frames:30});
assert.deepEqual(normalizeCutRange({min_cut_frames:30,max_cut_frames:10},'max_cut_frames'),{min_cut_frames:10,max_cut_frames:10});
assert.deepEqual(normalizeCutRange({min_cut_frames:30,max_cut_frames:10}),{min_cut_frames:10,max_cut_frames:30});
});
test("parameter aliases and mapping overlap rules preserve absolute mappings",()=>{
assert.equal(mappingTarget('base_brightness'),'brightness');
const rows=[{target:'dolly',start_frame:0,end_frame:48,enabled:true}];
assert.equal(mappingConflict(rows,{target:'dolly',start_frame:48,end_frame:null}),false);
assert.equal(mappingConflict(rows,{target:'dolly',start_frame:47,end_frame:null}),true);
assert.equal(mappingConflict(rows,{target:'dolly',start_frame:0,end_frame:null},0),false);
assert.equal(mappingConflict(rows,{target:'relief',start_frame:0,end_frame:null}),false);
assert.equal(mappingConflict(rows,{target:'dolly',start_frame:0,end_frame:null,enabled:false}),false);
});
test("previs controls have unique owners and depth movement respects strength",()=>{
const keys=Object.values(EFFECT_GROUPS).flat();assert.equal(keys.length,new Set(keys).size);
const s={motion_mode:"depth_parallax",parallax_strength:0,depth_relief:1,steady_depth:.5,orbit_degrees:0,dolly:.2,offset_x:.1,offset_y:0,scene_scale:1};
assert.deepEqual(demoPoint(.5,.5,.9,s,0),[.5,.5]);
assert.notDeepEqual(demoPoint(.5,.5,.9,{...s,parallax_strength:1},0),[.5,.5]);
assert.deepEqual(demoPoint(.5,.5,.5,{...s,parallax_strength:1},0),[.5,.5]);
});
test("timeline ranges are end-exclusive and dragging preserves length",()=>{
assert.deepEqual(mappingRange({start_frame:48,end_frame:null},192),[48,192]);
assert.deepEqual(moveMappingRange(48,96,-100,192),[0,48]);
assert.deepEqual(moveMappingRange(48,96,200,192),[144,192]);
});
test("seeks land inside a frame, not on the previous frame boundary", () => {
for(let frame=0;frame<192;frame++) {
const time=frameSeekTime(frame,24);
assert.ok(time>frame/24 && time<(frame+1)/24);
assert.equal(previewFrame(time,24,192),frame);
}
});
+295
View File
@@ -0,0 +1,295 @@
import importlib.util
from pathlib import Path
from types import SimpleNamespace
import unittest
from unittest.mock import Mock, patch
import torch
import comfy.sampler_helpers
spec = importlib.util.spec_from_file_location("fl_krea_test", Path(__file__).parents[1] / "nodes/conditioning/FL_KreaReference.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
def model():
result = SimpleNamespace(model=Mock(spec=m.comfy.model_base.Krea2), model_options={}, is_dynamic=lambda: False,
get_model_object=lambda name: SimpleNamespace(percent_to_sigma=lambda p: 1 - p))
result.clone = model
result.set_model_sampler_calc_cond_batch_function = lambda fn: result.model_options.update(sampler_calc_cond_batch_function=fn)
return result
def conditioning(value):
return [[torch.full((1, 2, 30720), float(value)), {"value": value}]]
def reference(**kwargs):
kwargs.setdefault("reference_mode", "full")
return m.FL_KreaReference.execute(torch.zeros(1, 48, 96, 3), **kwargs).result[0]
class KreaReferenceTests(unittest.TestCase):
def predict(self, branches, influence=.7, sigma=.5):
guider = m.KreaReferenceGuider(model(), conditioning(2), branches, influence)
guider.conds = guider.original_conds
guider.inner_model = object()
calls = []
def sample(inner, x, timestep, negative, positive, cfg, **kwargs):
calls.append(positive[0]["value"])
return torch.full_like(x, positive[0]["value"])
x = torch.zeros(1, 16, 2, 2)
with patch.object(m.comfy.samplers, "sampling_function", side_effect=sample):
result = guider.predict_noise(x, torch.tensor([sigma]))
return result, calls
def test_zero_influence_and_no_references_use_baseline(self):
for branches, strength in (([], .7), ([(reference(), conditioning(8))], 0)):
output, calls = self.predict(branches, strength)
self.assertEqual(calls, [2])
torch.testing.assert_close(output, torch.full_like(output, 2))
def test_single_reference_endpoint_skips_baseline(self):
output, calls = self.predict([(reference(), conditioning(8))], 1)
self.assertEqual(calls, [8])
torch.testing.assert_close(output, torch.full_like(output, 8))
def test_blend_formula_order_and_weight_scaling(self):
branches = [(reference(weight=.25), conditioning(4)), (reference(weight=.75), conditioning(8))]
a, calls = self.predict(branches, .5)
b, _ = self.predict(list(reversed(branches)), .5)
c, _ = self.predict([(dict(r, weight=r["weight"] * .5), cond) for r, cond in branches], .5)
self.assertEqual(calls, [2, 4, 8])
torch.testing.assert_close(a, torch.full_like(a, 4.5))
torch.testing.assert_close(a, b)
torch.testing.assert_close(c, torch.full_like(c, 3.25))
def test_single_reference_slider_scales_contribution(self):
for weight in (0, .05, .25, .5, 1):
with self.subTest(weight=weight):
output, _ = self.predict([(reference(weight=weight), conditioning(8))], .7)
torch.testing.assert_close(output, torch.full_like(output, 2 + .7 * weight * (8 - 2)))
def test_removing_reference_does_not_boost_remaining_reference(self):
a = (reference(weight=.25), conditioning(4))
b = (reference(weight=.5), conditioning(8))
both, _ = self.predict([a, b], 1)
only_b, _ = self.predict([b], 1)
zero_a, _ = self.predict([(reference(weight=0), conditioning(4)), b], 1)
torch.testing.assert_close(both - only_b, torch.full_like(both, .25 * (4 - 2)))
torch.testing.assert_close(only_b, zero_a)
def test_combined_weights_above_one_keep_unit_prediction_gain(self):
output, _ = self.predict([(reference(weight=1), conditioning(4)), (reference(weight=1), conditioning(4))], 1)
torch.testing.assert_close(output, torch.full_like(output, 6))
same, _ = self.predict([(reference(weight=1), conditioning(2)), (reference(weight=1), conditioning(2))], 1)
torch.testing.assert_close(same, torch.full_like(same, 2))
def test_schedule_does_not_renormalize_other_reference(self):
branches = [(reference(weight=.5, start=.6), conditioning(4)), (reference(weight=.5), conditioning(8))]
a, calls = self.predict(branches, 1, sigma=.7)
self.assertEqual(calls, [2, 8])
torch.testing.assert_close(a, torch.full_like(a, 5))
def test_envelope_boundaries_and_fades(self):
bounds = (.8, .6, .4, .2)
for sigma, expected in ((1, 0), (.8, 0), (.7, .5), (.6, 1), (.5, 1), (.3, .5), (.2, 0), (.1, 0)):
self.assertAlmostEqual(m.sigma_envelope(sigma, bounds), expected)
self.assertEqual(m.sigma_envelope(1, (1, 1, 0, 0)), 1)
self.assertEqual(m.sigma_envelope(0, (1, 1, 0, 0)), 1)
def test_repeat_evaluations_follow_sigma_not_call_count(self):
branches = [(reference(start=.25, end=.75), conditioning(8))]
for sigma in (.9, .5, .5, .1, .9):
output, _ = self.predict(branches, 1, sigma)
torch.testing.assert_close(output, torch.full_like(output, 8 if sigma == .5 else 2))
def test_reference_validation(self):
for kwargs in (dict(start=.5, end=.5), dict(start=-.1), dict(end=1.1), dict(fade=.6),
dict(weight=-1), dict(weight=1.01), dict(weight=10), dict(weight=float("nan")),
dict(weight=float("inf")), dict(role="unknown"), dict(resolution=999)):
with self.subTest(kwargs=kwargs), self.assertRaises(ValueError):
reference(**kwargs)
with self.assertRaisesRegex(ValueError, "one RGB image"):
m.FL_KreaReference.execute(torch.zeros(2, 32, 32, 3))
def clip(self):
clip = Mock()
clip.tokenizer = Mock(spec=m.Krea2Tokenizer)
clip.tokenize.side_effect = lambda text, **kwargs: (text, kwargs)
clip.encode_from_tokens_scheduled.side_effect = lambda tokens: conditioning(len(tokens[0]))
return clip
def test_disabled_and_zero_weight_references_are_not_encoded(self):
clip = self.clip()
with patch.object(m, "encode_reference", return_value=conditioning(8)) as encode:
guider = m.FL_KreaReferenceGuider.execute(model(), clip, "dog", references={
"reference_0": reference(enabled=False), "reference_1": reference(weight=0),
"reference_2": reference(),
}).result[0]
self.assertEqual(encode.call_count, 1)
self.assertEqual(len(guider.references), 1)
self.assertEqual(guider.references[0][1], 1)
def test_zero_strength_skips_reference_encoding(self):
clip = self.clip()
with patch.object(m, "encode_reference") as encode:
m.FL_KreaReferenceGuider.execute(model(), clip, "dog", influence=0, references={"reference_0": reference()})
encode.assert_not_called()
clip.tokenize.assert_called_once_with("dog")
def test_same_shape_images_and_clip_changes_are_reencoded(self):
clip = self.clip()
ref = reference()
encoded_images = []
def tokenize(text, **kwargs):
if "images" in kwargs:
encoded_images.append(kwargs["images"][0].clone())
return text, kwargs
clip.tokenize.side_effect = tokenize
for image in (torch.zeros_like(ref["image"]), torch.ones_like(ref["image"])):
m.FL_KreaReferenceGuider.execute(model(), clip, "dog", references={"reference_0": dict(ref, image=image)})
other_clip = self.clip()
m.FL_KreaReferenceGuider.execute(model(), other_clip, "dog", references={"reference_0": ref})
self.assertEqual(len(encoded_images), 2)
self.assertFalse(torch.equal(*encoded_images))
self.assertEqual(other_clip.encode_from_tokens_scheduled.call_count, 2)
def test_each_branch_sees_one_image_and_preserves_taps(self):
clip = self.clip()
refs = {"reference_0": reference(), "reference_1": reference(role="palette")}
guider = m.FL_KreaReferenceGuider.execute(model(), clip, "dog", references=refs).result[0]
calls = clip.tokenize.call_args_list
self.assertEqual(len(calls), 3)
for call in calls[1:]:
self.assertEqual(len(call.kwargs["images"]), 1)
self.assertEqual(call.args[0].count("<|image_pad|>"), 1)
self.assertIn("dog", call.args[0])
self.assertIn(m.ROLES["palette"], calls[2].args[0])
for cond in guider.original_conds.values():
self.assertEqual(cond[0]["cross_attn"].shape[-1], 30720)
self.assertTrue(torch.all(cond[0]["cross_attn"] != 0))
def test_model_and_guidance_mismatch_rejected_before_encoding(self):
clip = self.clip()
bad = model()
bad.model = object()
with self.assertRaisesRegex(ValueError, "Krea 2"):
m.FL_KreaReferenceGuider.execute(bad, clip, "dog")
bad = model()
bad.model_options["sampler_cfg_function"] = lambda args: None
with self.assertRaisesRegex(ValueError, "guidance patches"):
m.FL_KreaReferenceGuider.execute(bad, clip, "dog")
clip.encode_from_tokens_scheduled.assert_not_called()
def test_schema_has_optional_growing_references(self):
schema = m.FL_KreaReferenceGuider.INPUT_TYPES()
self.assertEqual(schema["optional"]["references"][1]["template"]["min"], 0)
def test_ksampler_outputs_preserve_guider_and_original_model(self):
original = model()
guider, patched, cond = m.FL_KreaReferenceGuider.execute(original, self.clip(), "dog", references={
"reference_0": reference(weight=.25), "reference_1": reference(weight=.5),
}).result
self.assertIsInstance(guider, m.KreaReferenceGuider)
self.assertEqual(original.model_options, {})
self.assertIs(patched.model_options["sampler_calc_cond_batch_function"], m.reference_cond_batch)
self.assertEqual(len(cond), 3)
self.assertNotIn("fl_krea_reference", cond[0][1])
self.assertEqual(cond[1][1]["fl_krea_reference"][1], .7 * .25)
def test_ksampler_matches_guider_weights_schedules_and_cfg(self):
x = torch.zeros(1, 16, 2, 2)
def batch(inner, conds, x, sigma, options):
return [torch.full_like(x, cond[0]["value"] if cond else 0) for cond in conds]
for weight in (0, .05, .5, 1):
for sigma in (.9, .5, .1):
refs = [reference(weight=weight, start=.2, end=.8, fade=.25), reference(weight=.3)]
branches = list(zip(refs, (conditioning(8), conditioning(4))))
expected, _ = self.predict(branches, .7, sigma)
clip = self.clip()
clip.encode_from_tokens_scheduled.side_effect = lambda tokens: conditioning(2)
with patch.object(m, "encode_reference", side_effect=[cond for ref, cond in branches if ref["weight"] > 0]):
_, patched, cond = m.FL_KreaReferenceGuider.execute(model(), clip, "dog", references=dict(enumerate(refs))).result
prepared = comfy.sampler_helpers.convert_cond(cond)
negative = comfy.sampler_helpers.convert_cond(conditioning(-2))
with patch.object(m.comfy.samplers, "calc_cond_batch", side_effect=batch):
for cfg in (1., 2.):
actual = m.comfy.samplers.sampling_function(object(), x, torch.tensor([sigma]), negative, prepared, cfg, model_options=patched.model_options)
torch.testing.assert_close(actual, -2 + (expected + 2) * cfg)
def test_batch_hook_leaves_plain_and_missing_conditioning_unchanged(self):
cond = [{"value": 2}]
with patch.object(m.comfy.samplers, "calc_cond_batch", side_effect=lambda model, conds, *args: conds):
out = m.reference_cond_batch(dict(model=object(), conds=[cond, None], input=None, sigma=torch.tensor([.5]), model_options={}))
self.assertEqual(out, [cond, None])
def test_average_blends_predictions_identically_for_both_sampler_outputs(self):
cases = [
([], [], 2, 2),
([reference()], [8], 8, 8),
([reference() for _ in range(4)], [4, 6, 8, 10], 22, 7),
([reference(), reference(weight=.5)], [4, 8], 7, 4.5),
([reference(weight=0), reference()], [4, 8], 8, 5),
([reference(enabled=False), reference()], [4, 8], 8, 8),
([reference(start=.6), reference()], [4, 8], 8, 5),
([reference(start=.25, end=.75, fade=.5), reference()], [4, 8], 10, 6),
]
x = torch.zeros(1, 16, 2, 2)
for refs, values, additive, average in cases:
for mode in ("add", "average"):
for amount in (0, .5, 1):
for influence in (0, .7, 1):
with self.subTest(values=values, mode=mode, amount=amount, influence=influence):
clip = self.clip()
clip.encode_from_tokens_scheduled.side_effect = lambda tokens: conditioning(2)
encoded = [conditioning(value) for ref, value in zip(refs, values) if ref["enabled"] and ref["weight"] > 0]
with patch.object(m, "encode_reference", side_effect=encoded):
guider, patched, cond = m.FL_KreaReferenceGuider.execute(
model(), clip, "dog", influence, dict(enumerate(refs)), mode, amount).result
guider.conds = guider.original_conds
guider.inner_model = object()
with patch.object(m.comfy.samplers, "sampling_function", side_effect=lambda inner, x, t, neg, pos, cfg, **kwargs: torch.full_like(x, pos[0]["value"])):
guided = guider.predict_noise(x, torch.tensor([.5]))
with patch.object(m.comfy.samplers, "calc_cond_batch", side_effect=lambda inner, conds, x, sigma, options: [torch.full_like(x, cond[0]["value"] if cond else 0) for cond in conds]):
sampled = m.comfy.samplers.sampling_function(
object(), x, torch.tensor([.5]), None, comfy.sampler_helpers.convert_cond(cond), 1,
model_options=patched.model_options)
blended = additive + (average - additive) * amount if mode == "average" else additive
expected = torch.full_like(x, 2 + influence * (blended - 2))
torch.testing.assert_close(guided, expected)
torch.testing.assert_close(sampled, expected)
def test_invalid_blend_controls_fail_before_encoding(self):
clip = self.clip()
for kwargs in (dict(blend_mode="unknown"), dict(average_amount=-.1), dict(average_amount=1.1), dict(average_amount=float("nan"))):
with self.subTest(kwargs=kwargs), self.assertRaises(ValueError):
m.FL_KreaReferenceGuider.execute(model(), clip, "dog", **kwargs)
clip.encode_from_tokens_scheduled.assert_not_called()
def test_context_mode_removes_visual_prefix_and_copies_suffix(self):
clip = self.clip()
clip.tokenize.side_effect = None
clip.tokenize.return_value = {"qwen3vl_4b": [[(151652, 1), ({"type": "image"}, 1), (151653, 1), (12, 1), (13, 1)]]}
tensor = torch.randn(1, 12, 30720)
mask = torch.ones(1, 12)
clip.encode_from_tokens_scheduled.side_effect = None
clip.encode_from_tokens_scheduled.return_value = [[tensor, {"attention_mask": mask, "other": 7}]]
result = m.encode_reference(clip, "dog", reference(reference_mode="context"))
torch.testing.assert_close(result[0][0], tensor[:, -2:])
self.assertEqual(result[0][0].shape[-1], 30720)
self.assertNotEqual(result[0][0].untyped_storage().data_ptr(), tensor.untyped_storage().data_ptr())
self.assertEqual(result[0][1]["attention_mask"].shape, (1, 2))
self.assertEqual(result[0][1]["other"], 7)
self.assertEqual(mask.shape, (1, 12))
if __name__ == "__main__":
unittest.main()
+58
View File
@@ -0,0 +1,58 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
import vm from "node:vm";
const source = await readFile(new URL("../web/nodes/conditioning/FL_KreaReference.js", import.meta.url), "utf8");
let extension;
vm.runInNewContext(source.replace(/^import .*;$/gm, ""), {
app: { registerExtension: value => { extension = value; } },
});
test("average slider follows the selected blend mode on change and load", () => {
const mode = { name: "blend_mode", value: "add" };
const amount = { name: "average_amount", value: .6 };
const node = {
constructor: { comfyClass: "FL_KreaReferenceGuider" },
widgets: [mode, amount], setDirtyCanvas() {},
};
extension.nodeCreated(node);
assert.equal(amount.disabled, true);
mode.value = "average";
mode.callback();
assert.equal(amount.disabled, false);
assert.equal(amount.value, .6);
mode.value = "add";
node.onConfigure();
assert.equal(amount.disabled, true);
node.inputs = [{ name: "blend_mode", link: 12 }];
node.onConnectionsChange();
assert.equal(amount.disabled, false);
});
test("removing instruction preserves saved reference settings and connected inputs", () => {
for (const objectLinks of [false, true]) {
const graph = {
nodes: [
{ id: 1, outputs: [{ links: [10, 11] }] },
{ id: 2, type: "FL_KreaReference",
widgets_values: [true, "palette", "Old instruction", .35, 1024, .1, .9, .2, "full"],
widgets_values_named: { instruction: "Old instruction", weight: .35 },
inputs: [{ name: "image", link: null }, { name: "instruction", link: 10 }, { name: "weight", link: 11 }] },
],
links: [[10, 1, 0, 2, 1, "STRING"], [11, 1, 0, 2, 2, "FLOAT"]],
};
if (objectLinks) graph.links = graph.links.map(([id, origin_id, origin_slot, target_id, target_slot, type]) =>
({ id, origin_id, origin_slot, target_id, target_slot, type }));
extension.beforeConfigureGraph(graph);
assert.deepEqual(graph.nodes[1].widgets_values, [true, "palette", .35, 1024, .1, .9, .2, "full"]);
assert.deepEqual(graph.nodes[1].widgets_values_named, { weight: .35 });
assert.deepEqual(graph.nodes[0].outputs[0].links, [11]);
assert.equal(graph.links.length, 1);
assert.equal(objectLinks ? graph.links[0].target_slot : graph.links[0][4], 1);
assert.deepEqual(graph.nodes[1].inputs.map(input => input.name), ["image", "weight"]);
const migrated = JSON.stringify(graph);
extension.beforeConfigureGraph(graph);
assert.equal(JSON.stringify(graph), migrated);
}
});
+78
View File
@@ -0,0 +1,78 @@
import importlib
from pathlib import Path
import sys
from types import ModuleType, SimpleNamespace
import unittest
from unittest.mock import patch
import torch
import comfy.latent_formats
import comfy.sample
package = ModuleType("fl_plus_test")
package.__path__ = [str(Path(__file__).parents[1] / "nodes/ksamplers")]
sys.modules[package.__name__] = package
m = importlib.import_module("fl_plus_test.FL_KsamplerPlus")
class KsamplerPlusTests(unittest.TestCase):
def sample(self, source, latent_format, x_slices=1, y_slices=1, batch_size=1, overlap=0, **metadata):
objects = {"latent_format": latent_format, "diffusion_model.vae_scale_factors": (4, 8, 8)}
model = SimpleNamespace(get_model_object=objects.__getitem__)
calls = []
def sampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent, **kwargs):
calls.append(latent["samples"].shape)
native = comfy.sample.fix_empty_latent_channels(model, latent["samples"])
return ({"samples": native + 1},)
with patch.object(m.comfy.model_management, "get_torch_device", return_value=torch.device("cpu")), \
patch.object(m, "common_ksampler", side_effect=sampler):
result = m.FL_KsamplerPlus().sample(
model, [], [], 42, 8, 1, "euler", "simple", 1, "latent",
x_slices, y_slices, overlap, batch_size, False,
latent_image={"samples": source, **metadata})[3]["samples"]
return result, calls
def test_empty_image_latent_uses_native_krea_shape_before_sampling(self):
source = torch.zeros(1, 4, 128, 128)
result, calls = self.sample(source, comfy.latent_formats.Wan21(), downscale_ratio_spacial=8)
self.assertEqual(calls, [torch.Size((1, 16, 1, 128, 128))])
self.assertEqual(result.shape, (1, 16, 1, 128, 128))
torch.testing.assert_close(result, torch.ones_like(result))
self.assertEqual(source.shape, (1, 4, 128, 128))
self.assertEqual(source.count_nonzero(), 0)
def test_tiles_preserve_native_dimensions_and_image_batches(self):
for shape, latent_format in (((2, 16, 16, 16), comfy.latent_formats.Flux()),
((2, 16, 16, 16), comfy.latent_formats.Wan21()),
((2, 16, 3, 16, 16), comfy.latent_formats.Wan21())):
for batch_size in (1, 4):
with self.subTest(shape=shape, batch_size=batch_size):
source = torch.rand(shape) + 1
expected = comfy.sample.fix_empty_latent_channels(
SimpleNamespace(get_model_object=lambda name: latent_format), source) + 1
result, calls = self.sample(source, latent_format, 2, 2, batch_size, .25)
torch.testing.assert_close(result, expected)
self.assertEqual(len(calls), 4 // batch_size)
self.assertTrue(all(shape[-2:] == (10, 10) for shape in calls))
def test_empty_latent_scale_metadata_is_applied_before_tiling(self):
source = torch.zeros(1, 4, 16, 16)
result, calls = self.sample(source, comfy.latent_formats.Wan21(), 2, 2,
downscale_ratio_spacial=16, downscale_ratio_temporal=8)
self.assertEqual(result.shape, (1, 16, 2, 32, 32))
self.assertEqual(calls, [torch.Size((1, 16, 2, 16, 16))] * 4)
torch.testing.assert_close(result, torch.ones_like(result))
def test_image_noise_mask_can_be_cropped_with_native_5d_latent(self):
source = torch.ones(1, 16, 16, 16)
mask = torch.ones(1, 1, 16, 16)
result, calls = self.sample(source, comfy.latent_formats.Wan21(), 2, 2, noise_mask=mask)
self.assertEqual(result.shape, (1, 16, 1, 16, 16))
torch.testing.assert_close(result, torch.full_like(result, 2))
if __name__ == "__main__":
unittest.main()
+208
View File
@@ -0,0 +1,208 @@
import importlib.util
from pathlib import Path
import unittest
from unittest.mock import patch
import torch
from comfy_api.latest import _io
spec = importlib.util.spec_from_file_location("parallax_test", Path(__file__).parents[1] / "nodes/vfx/FL_LayeredParallax.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
def plate(color, depth, count=3):
image = torch.zeros(count, 32, 64, 4)
image[:, 8:24, 24:40, :3] = torch.tensor(color)
image[:, 8:24, 24:40, 3] = 1
return m.FL_ParallaxLayer.execute(image, depth, 1, 0, 0, 1).result[0]
def render(layers, **kwargs):
values = dict(background=torch.zeros(3, 32, 64, 3), width=64, height=32, motion="glide", travel_x=.3,
travel_y=0, push_in=0, background_depth=12, overscan=1, device="cpu", layers=layers)
values.update(kwargs)
with patch.object(m, "preview_layers", return_value=[]):
return m.FL_LayeredParallax.execute(**values).result
class ParallaxTests(unittest.TestCase):
def test_rgba_batch_stack_preserves_order_alpha_and_variable_count(self):
for count in (1, 2, 4, 8):
images = torch.rand(count, 16, 16, 4)
stack = m.FL_ParallaxStackFromBatch.execute(images, 2, 10).result[0]
self.assertEqual(len(stack["layers"]), count - 1)
torch.testing.assert_close(stack["background"], images[:1])
for i, layer in enumerate(stack["layers"]):
torch.testing.assert_close(layer["images"], images[i + 1:i + 2])
self.assertNotEqual(layer["images"].untyped_storage().data_ptr(), images.untyped_storage().data_ptr())
depths = [p["depth"] for p in stack["layers"]]
self.assertEqual(depths, sorted(depths, reverse=True))
if depths:
self.assertEqual(depths[-1], 2)
def test_rgba_batch_stack_rejects_rgb_and_inverted_depths(self):
for images, near, far in ((torch.ones(2, 16, 16, 3), 2, 10), (torch.ones(2, 16, 16, 4), 10, 2)):
with self.assertRaises(ValueError):
m.FL_ParallaxStackFromBatch.execute(images, near, far)
def test_rgba_batch_stack_connects_to_depth_and_compositor(self):
images = torch.zeros(3, 32, 64, 4)
images[0, ..., 3] = 1
images[1, 4:20, 4:20, 0] = 1
images[1, 4:20, 4:20, 3] = 1
images[2, 12:28, 12:28, 1] = 1
images[2, 12:28, 12:28, 3] = 1
stack = m.FL_ParallaxStackFromBatch.execute(images, 2, 10).result[0]
analysis = m.FL_ParallaxDepthSources.execute(stack, 126).result[0]
self.assertEqual(len(analysis), 3)
_, neutral, _ = render({}, layer_stack=stack, frames=3, motion="locked")
expected = images[0:1, ..., :3]
for layer in images[1:]:
expected = layer[None, ..., :3] * layer[None, ..., 3:4] + expected * (1 - layer[None, ..., 3:4])
torch.testing.assert_close(neutral[:1], expected)
def test_animated_sources_keep_per_frame_neutral_composites(self):
background = torch.rand(9, 32, 64, 3)
p = plate([.2, .5, .8], 2, count=9)
p["images"][:, 8:24, 24:40, 0] = torch.arange(9)[:, None, None] / 9
_, locked, _ = render({"p": p}, background=background)
self.assertNotEqual(locked.stride(0), 0)
for i in (0, 4, 8):
still = dict(p, images=p["images"][i:i+1])
expected = render({"p": still}, background=background[i:i+1], frames=1, motion="locked")[1]
torch.testing.assert_close(locked[i:i+1], expected)
def test_static_locked_output_shares_one_frame(self):
p = plate([1, 0, 0], 2, count=1)
moving, locked, _ = render({"p": p}, background=torch.zeros(1, 32, 64, 3), frames=13)
self.assertEqual(locked.stride(0), 0)
self.assertEqual(locked.untyped_storage().nbytes(), 32 * 64 * 3 * 4)
for frame in locked:
torch.testing.assert_close(frame, locked[0])
self.assertFalse(torch.equal(moving[0], moving[-1]))
def test_contain_preserves_mismatched_cutout_aspect(self):
image = torch.ones(1, 64, 64, 4)
p = m.FL_ParallaxLayer.execute(image, 2, 1, 0, 0, 1).result[0]
cover = render({"p": p}, motion="locked", layer_fit="cover")[0]
contain = render({"p": p}, motion="locked", layer_fit="contain")[0]
self.assertEqual(float(cover[0, 16, 2, 0]), 1)
self.assertEqual(float(contain[0, 16, 2, 0]), 0)
self.assertEqual(float(contain[0, 16, 32, 0]), 1)
def test_thumbnail_metadata_and_explicit_alpha(self):
p = plate([1, 0, 0], 2, count=1)
p["name"] = "Subject"
with patch.object(m.nodes.SaveImage, "save_images", return_value={"ui": {"images": [{"filename": "thumb.png", "type": "output"}]}}) as save:
previews = m.preview_layers(torch.zeros(1, 512, 1024, 3), [p])
self.assertEqual(len(previews), 2)
self.assertEqual(previews[1]["name"], "Subject")
self.assertEqual(save.call_args_list[0].args[0].shape, (1, 128, 256, 4))
self.assertTrue(previews[0]["background"])
def test_dynamic_stack_matches_explicit_layers(self):
p = plate([1, 0, 0], 2)
background = torch.zeros(3, 32, 64, 3)
expected = render({"layer": p}, background=background)
actual = render({}, background=None, layer_stack={"background": background, "layers": [p]})
for a, b in zip(actual, expected):
torch.testing.assert_close(a, b)
def test_autogrow_wire_names_normalize_to_layers(self):
values = {"layers.layer_0": "first", "layers.layer_1": "second"}
_, _, data = _io.get_finalized_class_inputs(m.FL_LayeredParallax.INPUT_TYPES(), values)
self.assertEqual(_io.build_nested_inputs(values, data), {"layers": {"layer_0": "first", "layer_1": "second"}})
def test_depth_sort_and_locked_identity(self):
out, flat, depth = render({"near": plate([1, 0, 0], 2), "far": plate([0, 1, 0], 5)}, motion="locked")
torch.testing.assert_close(out, flat)
torch.testing.assert_close(out[:, 16, 32], torch.tensor([[1., 0, 0]]).expand(3, -1))
self.assertEqual(depth.shape, (1, 32, 64, 3))
def test_near_layer_moves_faster(self):
def displacement(depth):
out = render({"layer": plate([1, 0, 0], depth)})[0]
weights = out[..., 0].sum(1)
center = (weights * torch.arange(64)).sum(1) / weights.sum(1)
return float((center[-1] - center[0]).abs())
self.assertAlmostEqual(displacement(2) / displacement(4), 2, places=3)
def test_input_unchanged_and_output_finite(self):
p = plate([.2, .5, .9], 2)
original = p["images"].clone()
out, flat, _ = render({"a": p}, push_in=.1)
torch.testing.assert_close(p["images"], original)
self.assertTrue(torch.isfinite(out).all() and (out >= 0).all() and (out <= 1).all())
self.assertFalse(torch.equal(out, flat))
def test_still_layer_broadcasts(self):
out = render({"a": plate([1, 0, 0], 2, count=1)})[0]
self.assertEqual(len(out), 3)
def test_still_plates_generate_requested_video_length(self):
out, flat, _ = render({"a": plate([1, 0, 0], 2, count=1)}, background=torch.zeros(1, 32, 64, 3), frames=12)
self.assertEqual(len(out), 12)
torch.testing.assert_close(flat[0], flat[-1])
self.assertFalse(torch.equal(out[0], out[-1]))
def test_explicit_length_rejects_mismatched_video(self):
with self.assertRaisesRegex(ValueError, "equal frame counts"):
render({}, frames=12)
def test_frame_mismatch_fails_clearly(self):
with self.assertRaisesRegex(ValueError, "equal frame counts"):
render({"a": plate([1, 0, 0], 2, count=2)})
def test_explicit_mask_white_is_foreground(self):
rgb = torch.ones(1, 32, 64, 3)
mask = torch.zeros(1, 32, 64)
mask[:, 8:24, 24:40] = .5
p = m.FL_ParallaxLayer.execute(rgb, 2, 1, 0, 0, 1, mask=mask).result[0]
out = render({"a": p}, motion="locked")[0]
self.assertAlmostEqual(float(out[0, 16, 32, 0]), .5)
self.assertEqual(float(out[0, 0, 0, 0]), 0)
def test_mask_override_does_not_multiply_embedded_alpha(self):
rgba = torch.ones(1, 32, 64, 4) * .5
mask = torch.ones(1, 32, 64)
p = m.FL_ParallaxLayer.execute(rgba, 2, 1, 0, 0, 1, mask=mask).result[0]
self.assertAlmostEqual(float(render({"a": p}, motion="locked")[0][0, 16, 32, 0]), .5)
def test_empty_layers_and_single_frame(self):
bg = torch.rand(1, 32, 64, 3)
out = render({}, background=bg)[0]
torch.testing.assert_close(out, bg, atol=1e-6, rtol=1e-6)
def test_premultiplied_sampling_has_no_white_fringe(self):
rgba = torch.ones(1, 32, 64, 4)
rgba[..., 3] = 0
rgba[:, 8:24, 24:40, :3] = torch.tensor([1., 0, 0])
rgba[:, 8:24, 24:40, 3] = 1
p = m.FL_ParallaxLayer.execute(rgba, 2, 1.1, .017, 0, 1).result[0]
out = render({"a": p})[0]
self.assertEqual(float(out[..., 1:].max()), 0)
def test_no_alpha_and_invalid_mask_rejected(self):
with self.assertRaisesRegex(ValueError, "no cutout alpha"):
m.FL_ParallaxLayer.execute(torch.ones(1, 32, 64, 3), 2, 1, 0, 0, 1)
with self.assertRaisesRegex(ValueError, "match image size"):
m.FL_ParallaxLayer.execute(torch.ones(1, 32, 64, 3), 2, 1, 0, 0, 1, mask=torch.ones(1, 8, 8))
def test_camera_paths(self):
self.assertEqual(m.camera_path(1, "bursts"), [0])
self.assertEqual(m.camera_path(124, "bursts")[0], -1)
self.assertEqual(m.camera_path(124, "bursts")[-1], 1)
self.assertAlmostEqual(m.camera_path(124, "loop")[0], m.camera_path(124, "loop")[-1])
@unittest.skipUnless(torch.cuda.is_available(), "CUDA unavailable")
def test_gpu_cpu_agree(self):
p = plate([.2, .4, .9], 2)
cpu = render({"a": p})[0]
gpu = render({"a": p}, device="auto")[0]
torch.testing.assert_close(cpu, gpu, atol=1e-5, rtol=1e-5)
if __name__ == "__main__":
unittest.main()
+76
View File
@@ -0,0 +1,76 @@
import importlib.util
import json
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import torch
from safetensors import safe_open
from safetensors.torch import load_file
spec = importlib.util.spec_from_file_location("fl_lora", Path(__file__).parents[1] / "nodes/utility/FL_ModelDifferenceLoraSave.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
def model(weight, bias=None):
network = torch.nn.Module()
network.diffusion_model = torch.nn.Module()
network.diffusion_model.linear = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=bias is not None)
network.diffusion_model.linear.weight = torch.nn.Parameter(weight.clone())
if bias is not None:
network.diffusion_model.linear.bias = torch.nn.Parameter(bias.clone())
return SimpleNamespace(model=network, backup={}, hook_backup={}, patches={})
class ModelDifferenceLoraTests(unittest.TestCase):
def test_export_roundtrip_sign_bias_and_no_input_mutation(self):
base = model(torch.randn(4, 5), torch.randn(4))
tuned = model(base.model.diffusion_model.linear.weight.detach() + torch.randn(4, 5),
base.model.diffusion_model.linear.bias.detach() + 0.25)
before = tuned.model.diffusion_model.linear.weight.detach().clone()
with TemporaryDirectory() as root, patch.object(m.folder_paths, "get_folder_paths", return_value=[root]):
path, = m.FL_ModelDifferenceLoraSave().save(tuned, base, "Krea2/test", 4, "cpu")
sd = load_file(path)
key = "diffusion_model.linear"
delta = sd[key + ".lora_up.weight"].float() @ sd[key + ".lora_down.weight"].float()
torch.testing.assert_close(base.model.diffusion_model.linear.weight + delta, before, atol=.003, rtol=.003)
torch.testing.assert_close(sd[key + ".bias.diff"].float(), torch.full((4,), .25))
key_map = {key: key + ".weight", key + ".bias": key + ".bias"}
patches = m.comfy.lora.load_lora(sd, key_map)
self.assertEqual(set(patches), set(key_map.values()))
with safe_open(path, framework="pt") as f:
self.assertAlmostEqual(json.loads(f.metadata()["fl_extraction"])["retained_weight_energy"], 1, places=5)
second, = m.FL_ModelDifferenceLoraSave().save(tuned, base, "Krea2/test", 4, "cpu")
self.assertNotEqual(path, second)
torch.testing.assert_close(tuned.model.diffusion_model.linear.weight, before)
def test_lowrank_and_convolution_reconstruction(self):
for shape in ((96, 80), (96, 5, 4, 4)):
diff = (torch.randn(96, 3) @ torch.randn(3, 80)).reshape(shape)
up, down = m.factorize_difference(diff, 3)
torch.testing.assert_close(up.flatten(1) @ down.flatten(1), diff.flatten(1), atol=1e-4, rtol=1e-4)
def test_patches_and_backup_are_applied_before_subtraction(self):
target = model(torch.full((3, 4), 99.))
key = "diffusion_model.linear.weight"
target.backup[key] = SimpleNamespace(weight=torch.ones(3, 4))
target.patches[key] = [(0.5, (torch.full((3, 4), 4.),), 1., None, None)]
torch.testing.assert_close(m.materialize_weight(target, key, torch.device("cpu")), torch.full((3, 4), 3.))
def test_invalid_models_and_paths_do_not_save(self):
base = model(torch.zeros(4, 5))
with TemporaryDirectory() as root, patch.object(m.folder_paths, "get_folder_paths", return_value=[root]):
for tuned, prefix in ((base, "same"), (model(torch.zeros(3, 5)), "shape"),
(model(torch.full((4, 5), float("nan"))), "nan"),
(base, "../escape"), (base, "C:/escape")):
with self.subTest(prefix=prefix), self.assertRaises(ValueError):
m.FL_ModelDifferenceLoraSave().save(tuned, base, prefix, 4, "cpu")
self.assertEqual(list(Path(root).rglob("*.safetensors")), [])
if __name__ == "__main__":
unittest.main()
+32
View File
@@ -0,0 +1,32 @@
import assert from "node:assert/strict";
import {readFile} from "node:fs/promises";
import test from "node:test";
const source=await readFile(new URL("../web/nodes/vfx/parallax_preview.js",import.meta.url),"utf8");
const {cameraValue,plateRect,viewport,CAMERA_PRESETS}=await import(`data:text/javascript;base64,${Buffer.from(source).toString("base64")}`);
const settings={width:640,height:400,motion:"glide",travel_x:.3,travel_y:0,push_in:0,background_depth:12,overscan:1,layer_fit:"cover"};
const plate={width:640,height:400,depth:2,scale:1,offset_x:0,offset_y:0,opacity:1};
test("camera curves match render anchors",()=>{
assert.equal(cameraValue(.25,"loop"),1);
assert.equal(cameraValue(.5,"glide"),0);
assert.equal(cameraValue(0,"glide"),-1);
assert.equal(cameraValue(1,"glide"),1);
for(const [t,v] of [[0,-1],[.1,-1],[.28,.4],[.45,.4],[.61,-.3],[.73,-.3],[.92,1],[1,1]])assert.ok(Math.abs(cameraValue(t,"bursts")-v)<1e-12);
assert.equal(cameraValue(.3,"locked"),0);
});
test("near plates move twice as much at half the depth",()=>{
const near=plateRect(plate,settings,1,640,400),far=plateRect({...plate,depth:4},settings,1,640,400);
assert.equal(near.x,2*far.x);
assert.equal(near.x,-96);
});
test("neutral projection and contain preserve aspect",()=>{
assert.deepEqual(plateRect(plate,settings,.5,640,400),{x:0,y:0,w:640,h:400});
const result=plateRect({...plate,width:400},{...settings,layer_fit:"contain"},.5,640,400);
assert.deepEqual(result,{x:120,y:0,w:400,h:400});
assert.deepEqual(viewport(480,360,.75),{x:105,y:0,w:270,h:360});
});
test("background uses camera framing, presets only change motion",()=>{
const p=plateRect({...plate,background:true},{...settings,overscan:1.2},.5,640,400);
assert.equal(p.w,768);
assert.equal(p.h,480);
for(const preset of Object.values(CAMERA_PRESETS))assert.deepEqual(Object.keys(preset),["motion","travel_x","travel_y","push_in"]);
});
+59
View File
@@ -0,0 +1,59 @@
import unittest
import torch
from test_layered_parallax import m, plate, render
class ReliefTests(unittest.TestCase):
def test_zero_strength_matches_original(self):
p = plate([1, 0, 0], 3, 1)
a = render({'layer_0': p}, background=torch.zeros(1,32,64,3), frames=5)
b = render({'layer_0': p}, background=torch.zeros(1,32,64,3), frames=5, relief_scope='background', relief_strength=0)
for x, y in zip(a, b):
torch.testing.assert_close(x, y, rtol=0, atol=0)
def test_artwork_relief_changes_motion_not_locked(self):
p = plate([1, 0, 0], 3, 1)
bg = torch.zeros(1, 32, 64, 3)
depth = torch.ones(2, 32, 64, 3)
a = render({'layer_0': p}, background=bg, frames=5)
b = render({'layer_0': p}, background=bg, frames=5, relief_scope='background + artwork', depth_maps=depth, relief_strength=.4)
self.assertGreater(float((a[0]-b[0]).abs().max()), .05)
torch.testing.assert_close(a[1], b[1], rtol=0, atol=0)
def test_text_stays_flat(self):
p = dict(plate([1, 0, 0], 3, 1), kind='text')
bg = torch.zeros(1, 32, 64, 3)
a = render({'layer_0': p}, background=bg, frames=5)[0]
b = render({'layer_0': p}, background=bg, frames=5, relief_scope='background + artwork', depth_maps=torch.ones(2, 32, 64, 3))[0]
torch.testing.assert_close(a, b, rtol=0, atol=0)
def test_neutral_depth_matches_flat_projection(self):
p = plate([1, 0, 0], 3, 1)
bg = torch.zeros(1, 32, 64, 3)
a = render({'layer_0': p}, background=bg, frames=5)[0]
b = render({'layer_0': p}, background=bg, frames=5, relief_scope='background + artwork', depth_maps=torch.full((2,32,64,3),.5), depth_smoothing=0)[0]
torch.testing.assert_close(a, b, atol=1e-5, rtol=1e-5)
def test_invalid_batch_rejected(self):
with self.assertRaisesRegex(ValueError, 'one background'):
render({}, background=torch.zeros(1,32,64,3), relief_scope='background', depth_maps=torch.zeros(3,32,64,3))
def test_source_batch_dynamic_and_alpha_neutralized(self):
stack = dict(background=torch.zeros(1,32,64,3), layers=[plate([1,0,0],3,1),plate([0,1,0],2,1)])
images=m.FL_ParallaxDepthSources.execute(stack,126).result[0]
self.assertEqual(images.shape[0],3)
self.assertEqual(images.shape[-1],3)
torch.testing.assert_close(images[1,0,0],torch.full((3,),.5))
def test_invert_and_smoothing(self):
d=torch.zeros(1,8,8,3);d[:,4:,4:]=1
a=m.prepare_relief(d,False,2,torch.device('cpu'))
b=m.prepare_relief(d,True,2,torch.device('cpu'))
torch.testing.assert_close(a+b,torch.ones_like(a))
self.assertTrue(torch.any((a>0)&(a<1)))
if __name__ == '__main__':
unittest.main()
+84
View File
@@ -0,0 +1,84 @@
import importlib.util
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import torch
from comfy_execution.cache_provider import CacheContext, CacheValue
from comfy_execution.cache_provider import register_cache_provider, unregister_cache_provider
from comfy_execution.caching import RAMPressureCache, CacheKeySetInputSignature
from comfy_execution.graph import DynamicPrompt, ExecutionList
import nodes
spec = importlib.util.spec_from_file_location("poster_cache_test", Path(__file__).parents[1] / "nodes/vfx/poster_layer_cache.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
class PosterCacheTests(unittest.IsolatedAsyncioTestCase):
async def test_expanded_asset_recovers_before_scheduling_ancestors(self):
poster_spec = importlib.util.spec_from_file_location("poster_integration_test", Path(__file__).parents[1] / "nodes/vfx/FL_PosterLayers.py")
poster = importlib.util.module_from_spec(poster_spec)
poster_spec.loader.exec_module(poster)
class Unchanged:
async def get(self, node_id):
return False
dyn = DynamicPrompt({"parent": {"class_type": "EmptyImage", "inputs": {}}})
dyn.add_ephemeral_node("extract", {"class_type": "EmptyImage", "inputs": {}}, "parent", "parent")
dyn.add_ephemeral_node("asset", {"class_type": "FL_PosterLayerAsset", "inputs": {"image": ["extract", 0], "layer_id": "layer_0"}}, "parent", "parent")
with tempfile.TemporaryDirectory() as root, patch.dict(nodes.NODE_CLASS_MAPPINGS, FL_PosterLayerAsset=poster.FL_PosterLayerAsset):
cache = RAMPressureCache(CacheKeySetInputSignature, enable_providers=True)
await cache.set_prompt(dyn, ["parent"], Unchanged())
await cache.ensure_subcache_for("parent", ["extract", "asset"])
provider = m.PosterLayerCache(root)
context = cache._build_context("asset", cache.cache_key_set.get_data_key("asset"))
expected = torch.rand(1, 8, 8, 4)
await provider.on_store(context, CacheValue(outputs=[[dict(image=expected)]]))
register_cache_provider(provider)
try:
execution = ExecutionList(dyn, cache)
execution.add_node("asset")
self.assertEqual(list(execution.pendingNodes), ["asset"])
restored = await cache.get("asset")
torch.testing.assert_close(restored.outputs[0][0]["image"], expected, atol=0, rtol=0)
self.assertTrue(execution.is_cached("asset"))
finally:
unregister_cache_provider(provider)
async def test_exact_disk_round_trip_and_key_invalidation(self):
with tempfile.TemporaryDirectory() as root:
cache = m.PosterLayerCache(root)
context = CacheContext("1", "FL_PosterLayerAsset", "a" * 64)
image = torch.rand(1, 8, 8, 4)
asset = dict(image=image, file={"filename": "test.png"}, thumbnail={}, coverage=.6)
self.assertIsNone(await cache.on_lookup(context))
await cache.on_store(context, CacheValue(outputs=[[asset]]))
restored = await m.PosterLayerCache(root).on_lookup(context)
torch.testing.assert_close(restored.outputs[0][0]["image"], image, rtol=0, atol=0)
self.assertEqual(restored.outputs[0][0]["file"], asset["file"])
self.assertIsNone(await cache.on_lookup(CacheContext("1", "FL_PosterLayerAsset", "b" * 64)))
async def test_scope_and_path_validation(self):
with tempfile.TemporaryDirectory() as root:
cache = m.PosterLayerCache(root)
for context in (CacheContext("1", "KSampler", "a" * 64), CacheContext("1", "FL_PosterLayerAsset", "../outside")):
self.assertFalse(cache.should_cache(context))
self.assertIsNone(await cache.on_lookup(context))
async def test_bounded_eviction_preserves_unrelated_files(self):
with tempfile.TemporaryDirectory() as root:
unrelated = Path(root) / "keep.txt"
unrelated.touch()
cache = m.PosterLayerCache(root, maximum_bytes=2000)
image = torch.zeros(1, 8, 8, 4)
for char in ("a", "b", "c"):
await cache.on_store(CacheContext("1", "FL_PosterLayerAsset", char * 64), CacheValue(outputs=[[dict(image=image)]]))
self.assertEqual(len(list(Path(root).glob("*.safetensors"))), 1)
self.assertTrue(unrelated.exists())
self.assertIsNotNone(await cache.on_lookup(CacheContext("1", "FL_PosterLayerAsset", "c" * 64)))
if __name__ == "__main__":
unittest.main()
+144
View File
@@ -0,0 +1,144 @@
import importlib.util
import json
from pathlib import Path
import unittest
from unittest.mock import patch
import torch
from comfy_api.latest import _io
spec = importlib.util.spec_from_file_location("poster_test", Path(__file__).parents[1] / "nodes/vfx/FL_PosterLayers.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
def plan(count):
return m.parse_plan(json.dumps([dict(name=f"Layer {i}", prompt=f"Object {i}", kind="art" if i else "background", depth=2 if i else 12) for i in range(count)]), count)
def graph(entries, edits=None):
m.GraphBuilder.set_default_prefix("test", 0)
settings = json.dumps({"plan_key": m.plan_key(entries), "layers": edits or {}})
return m.extraction_graph(entries, ["model", 0], ["clip", 0], ["vae", 0], ["image", 0], 777, 30, 4, "euler", "simple", settings).expand
class PosterTests(unittest.TestCase):
def test_auto_merges_duplicate_targets(self):
entries = plan(4)
entries[1]["prompt"] = "black speakers beside the woman"
entries[2]["prompt"] = "black speakers"
with patch.object(m.TextGenerate, "execute", return_value=m.io.NodeOutput(json.dumps(entries))):
result = m.FL_PosterLayerPlanner.execute(torch.zeros(1, 8, 8, 3), "auto", 4, "balanced", "[]", object())
self.assertEqual(len(result.result[0]), 3)
self.assertEqual([r["id"] for r in result.result[0]], ["layer_0", "layer_1", "layer_2"])
self.assertIn(" / ", result.result[0][1]["name"])
def test_auto_targets_do_not_include_other_object_anchors(self):
entries = plan(4)
entries[0]["prompt"] = "studio wall behind a table"
entries[1]["prompt"] = "large yellow disk behind the woman"
entries[2]["prompt"] = "silver stars positioned above the speakers"
entries[3]["prompt"] = "brass lamp beside a sofa"
with patch.object(m.TextGenerate, "execute", return_value=m.io.NodeOutput(json.dumps(entries))):
result = m.FL_PosterLayerPlanner.execute(torch.zeros(1, 8, 8, 3), "auto", 4, "balanced", "[]", object())
self.assertEqual([p["prompt"] for p in result.result[0]], ["studio wall behind a table", "large yellow disk", "silver stars", "brass lamp"])
manual = m.FL_PosterLayerPlanner.execute(torch.zeros(1, 8, 8, 3), "manual", 4, "balanced", json.dumps(entries))
self.assertEqual(manual.result[0][1]["prompt"], entries[1]["prompt"])
def test_auto_repairs_over_budget_plan_once(self):
responses = [m.io.NodeOutput(json.dumps(plan(4))), m.io.NodeOutput(json.dumps(plan(3)))]
with patch.object(m.TextGenerate, "execute", side_effect=responses) as generate:
result = m.FL_PosterLayerPlanner.execute(torch.zeros(1, 8, 8, 3), "auto", 3, "balanced", "[]", object(), "photography")
self.assertEqual(len(result.result[0]), 3)
self.assertEqual(generate.call_count, 2)
self.assertIn("photography", generate.call_args_list[0].args[1])
self.assertIn("Do not invent a headline", generate.call_args_list[0].args[1])
self.assertIn("returned 4 layers", generate.call_args_list[1].args[1])
def test_auto_repair_is_bounded(self):
with patch.object(m.TextGenerate, "execute", return_value=m.io.NodeOutput("[]")) as generate:
with self.assertRaises(ValueError):
m.FL_PosterLayerPlanner.execute(torch.zeros(1, 8, 8, 3), "auto", 3, "balanced", "[]", object())
self.assertEqual(generate.call_count, 2)
def test_asset_defers_extraction_until_after_cache_lookup(self):
image_input = m.FL_PosterLayerAsset.INPUT_TYPES()["required"]["image"]
self.assertTrue(image_input[1]["lazy"])
self.assertEqual(m.FL_PosterLayerAsset.check_lazy_status(), ["image"])
self.assertEqual(m.FL_PosterLayerAsset.check_lazy_status(image=torch.zeros(1, 8, 8, 4)), [])
def test_variable_layer_counts(self):
for count in (1, 3, 6, 12, 32):
g = graph(plan(count))
self.assertEqual(sum(n["class_type"] == "KSampler" for n in g.values()), count)
self.assertEqual(sum(n["class_type"] == "VAEEncode" for n in g.values()), 1)
self.assertEqual(sum(n["class_type"] == "EmptyQwenImageLayeredLatentImage" for n in g.values()), 1)
def test_layout_edits_do_not_change_sampling_graph(self):
p = plan(6)
old, new = graph(p), graph(p, {"layer_2": {"depth": 7, "visible": False, "scale": 1.2}})
changed = [key for key in old if old[key] != new[key]]
self.assertEqual(changed, ["test.0.0.stack"])
def test_reroll_changes_only_one_sampler(self):
p = plan(5)
old, new = graph(p), graph(p, {"layer_3": {"revision": 1}})
changed = [key for key in old if old[key] != new[key] and old[key]["class_type"] == "KSampler"]
self.assertEqual(changed, ["test.0.0.layer_3_sample"])
def test_prompt_edit_changes_only_target_conditioning(self):
p = plan(5)
old, new = graph(p), graph(p, {"layer_4": {"prompt": 'The red "AFTER HOURS" headline'}})
changed = [key for key in old if old[key] != new[key]]
self.assertEqual(set(changed), {"test.0.0.layer_4_text", "test.0.0.stack"})
def test_stale_overrides_not_applied_to_new_plan(self):
p = plan(3)
edits = json.dumps({"plan_key": m.plan_key(p), "layers": {"layer_1": {"prompt": "wrong old object"}}})
p[1]["prompt"] = "New artwork"
self.assertEqual(m.apply_overrides(p, edits)[1]["prompt"], "New artwork")
def test_bad_plan_rejected(self):
for value in ('not json', '{}', '[]', '[{"kind":"art","name":"a","prompt":"a"}]', '[{"kind":"background","name":"a","prompt":"a","depth":NaN}]'):
with self.assertRaises(ValueError):
m.parse_plan(value, 6)
with self.assertRaises(ValueError):
m.parse_plan(json.dumps(plan(4)), 3)
def test_fenced_thinking_output(self):
value = '<think>reasoning</think>\n```json\n' + json.dumps(plan(2)) + '\n```'
self.assertEqual(len(m.parse_plan(value, 6)), 2)
def test_bad_overrides_rejected(self):
p = plan(2)
for edit in ({"depth": float("nan")}, {"scale": 0}, {"prompt": ""}, {"visible": "false"}, {"revision": -1}):
with self.assertRaises(ValueError):
m.apply_overrides(p, json.dumps({"plan_key": m.plan_key(p), "layers": {"layer_1": edit}}))
def test_asset_path_validation_before_save(self):
with self.assertRaises(ValueError):
m.FL_PosterLayerAsset.execute(torch.ones(1, 8, 8, 4), "../../outside")
def test_autogrow_accepts_more_than_ten_assets(self):
values = {f"assets.asset_{i}": i for i in range(12)}
_, _, data = _io.get_finalized_class_inputs(m.FL_PosterLayerStack.INPUT_TYPES(), values)
self.assertEqual(len(_io.build_nested_inputs(values, data)["assets"]), 12)
def test_stack_visibility_and_review(self):
p = plan(3)
entries = m.apply_overrides(p, json.dumps({"plan_key": m.plan_key(p), "layers": {"layer_1": {"visible": False}}}))
assets = {f"asset_{i}": dict(image=torch.ones(1, 8, 8, 4), file={}, thumbnail={}, coverage=1) for i in range(3)}
out = m.FL_PosterLayerStack.execute(json.dumps(entries), m.plan_key(p), assets)
self.assertEqual(len(out.result[0]["layers"]), 1)
self.assertEqual(len(out.ui["poster_layers"]), 3)
self.assertEqual(out.result[0]["background"].shape, (1, 8, 8, 4))
def test_manual_mode_does_not_request_vision(self):
self.assertEqual(m.FL_PosterLayerPlanner.check_lazy_status(mode="manual"), [])
out = m.FL_PosterLayerPlanner.execute(torch.zeros(1, 8, 8, 3), "manual", 3, "balanced", json.dumps(plan(3)))
self.assertEqual(len(out.result[0]), 3)
if __name__ == "__main__":
unittest.main()
+23
View File
@@ -0,0 +1,23 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {ownsPosterEvent,posterLayerId,replacePreviewUrl} from '../web/nodes/vfx/poster_preview_events.js';
test('routes expanded nodes without matching unrelated node prefixes',()=>{
assert.equal(ownsPosterEvent(106,{node:'106.0.0.layer_2_asset'}),true);
assert.equal(ownsPosterEvent(106,{displayNodeId:'106',realNodeId:'106.0.0.layer_2_sample'}),true);
assert.equal(ownsPosterEvent(106,{node:'1060.0.layer_2_sample'}),false);
assert.equal(ownsPosterEvent(106,{node:null}),false);
assert.equal(ownsPosterEvent(106,{node_id:'106.0.0.layer_2_sample'}),true);
});
test('identifies sampling, decoding and completed layers',()=>{
for(const stage of ['sample','decode','asset'])assert.equal(posterLayerId({node:'106.0.0.layer_12_'+stage}),'layer_12');
assert.equal(posterLayerId({node:106}),null);
});
test('releases replaced previews and releases final URL on cleanup',()=>{
const revoked=[],state={url:null};let i=0;
const urls={createObjectURL:()=>`blob:${++i}`,revokeObjectURL:url=>revoked.push(url)};
assert.equal(replacePreviewUrl(state,{},urls),'blob:1');
assert.equal(replacePreviewUrl(state,{},urls),'blob:2');
assert.equal(replacePreviewUrl(state,null,urls),null);
assert.deepEqual(revoked,['blob:1','blob:2']);
});
+230
View File
@@ -0,0 +1,230 @@
import importlib
import importlib.util
import json
from pathlib import Path
import sys
import tempfile
import types
import unittest
import wave
from unittest.mock import patch
from PIL import Image
import torch
ROOT = Path(__file__).resolve().parents[1]
package = types.ModuleType("fl_reference_tests")
package.__path__ = [str(ROOT / "nodes" / "audio")]
sys.modules[package.__name__] = package
references = importlib.import_module("fl_reference_tests.prompt_references")
storyboards = importlib.import_module("fl_reference_tests.prompt_storyboards")
actions = importlib.import_module("fl_reference_tests.prompt_storyboard_actions")
spec = importlib.util.spec_from_file_location("h3_reference_test", ROOT.parent / "ComfyUI-FL-MiniMaxH3" / "nodes" / "_shot_references.py")
h3 = importlib.util.module_from_spec(spec)
spec.loader.exec_module(h3)
class ReferenceTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
for name in ("input", "output"):
(self.root / name).mkdir()
mock = patch.object(references.folder_paths, f"get_{name}_directory", return_value=str(self.root / name))
mock.start()
self.addCleanup(mock.stop)
self.store = storyboards.StoryboardStore(self.root / "jobs.db")
self.spec = {"scheduler_id": "project", "section_id": "section-a", "revision": "revision-a", "prompt": "Four chronological running poses", "grid": 2}
def test_legacy_defaults(self):
self.assertEqual(references.reference_document("", 2)["assets"], {})
defaults = (object(), object(), object(), object())
self.assertEqual(h3.resolve_shot_references([{}], None, *defaults)[:4], defaults)
def test_no_references_does_not_consume_defaults(self):
resolved = h3.resolve_shot_references([{"references": {"mode": "none"}}], None, 1, 2, 3, 4)
self.assertEqual(resolved, (None, None, None, None, []))
def test_selection_order_and_video_audio_pairing(self):
assets = {"a": {"kind": "image", "value": "image-a"}, "b": {"kind": "image", "value": "image-b"},
"v": {"kind": "video", "value": "video", "audio": "paired"}, "s": {"kind": "audio", "value": "sound"}}
section = {"references": {"mode": "custom", "asset_ids": ["b", "v", "a", "s"]}}
image, video, paired, audio, ids = h3.resolve_shot_references([section], {"version": 1, "assets": assets}, None, None, None, None)
self.assertEqual(list(image.values()), ["image-b", "image-a"])
self.assertEqual(video, {"ref_video_0": "video"})
self.assertEqual(paired, {"ref_video_audio_0": "paired"})
self.assertEqual(audio, {"ref_audio_0": "sound"})
self.assertEqual(ids, ["b", "v", "a", "s"])
def test_group_conflict_and_missing_library(self):
with self.assertRaisesRegex(ValueError, "Grouped render"):
h3.resolve_shot_references([{}, {"references": {"mode": "none"}}], None, None, None, None, None)
with self.assertRaisesRegex(ValueError, "Connect"):
h3.resolve_shot_references([{"references": {"mode": "custom", "asset_ids": ["a"]}}], None, None, None, None, None)
def test_path_containment(self):
for asset in ({"filename": "../secret"}, {"filename": "secret", "subfolder": "../../"},
{"filename": "secret", "type": "temp"}, {"filename": "C:\\secret"}):
with self.assertRaises(ValueError):
references.reference_path(asset, must_exist=False)
def test_reference_metadata_roundtrip(self):
value = {"version": 1, "assets": {"a": {"filename": "a.png", "kind": "image"}},
"sections": [{"id": "s1", "mode": "custom", "asset_ids": ["a"]}, {"id": "s2", "mode": "none", "asset_ids": []}]}
document = references.reference_document(json.dumps(value), 2)
sections = references.apply_reference_sections([{}, {}], document)
self.assertEqual(sections[0]["section_id"], "s1")
self.assertEqual(sections[1]["references"]["mode"], "none")
with self.assertRaisesRegex(ValueError, "match"):
references.reference_document(value, 1)
value["sections"][1]["id"] = "s1"
with self.assertRaisesRegex(ValueError, "unique"):
references.reference_document(value, 2)
def test_claim_is_one_shot_and_persists(self):
job = self.store.create(self.spec)
self.assertEqual(job["state"], "proposed")
graph = self.store.claim(job["id"])
self.assertEqual({node["class_type"] for node in graph.values()}, {"GeminiNanoBanana2V2", "SaveImage"})
with self.assertRaisesRegex(ValueError, "already submitted"):
storyboards.StoryboardStore(self.store.path).claim(job["id"])
self.assertEqual(len(self.store.list("project")), 1)
self.assertEqual(self.store.list("other-project"), [])
def test_cancel_prevents_submission(self):
job = self.store.create(self.spec)
self.store.update(job["id"], "cancelled", {})
with self.assertRaises(ValueError):
self.store.claim(job["id"])
def test_batch_branches_share_continuity_and_claim_atomically(self):
jobs = [self.store.create({**self.spec, "section_id": str(i), "continuity": "One pirate, crimson coat, hand-drawn anime."}) for i in range(3)]
graph = storyboards.storyboard_batch_graph(jobs)
self.assertEqual(sum(n["class_type"] == "GeminiNanoBanana2V2" for n in graph.values()), 3)
for job in jobs:
prefix = job["id"] + ":"
self.assertIn("One pirate", graph[prefix + "storyboard"]["inputs"]["prompt"])
self.assertEqual(graph[prefix + "save_storyboard"]["inputs"]["images"], [prefix + "storyboard", 0])
self.store.update(jobs[-1]["id"], "cancelled", {})
with self.assertRaises(ValueError):
self.store.claim_batch([job["id"] for job in jobs])
self.assertEqual(self.store.get(jobs[0]["id"])["state"], "proposed")
self.store.claim_batch([job["id"] for job in jobs[:2]])
self.assertEqual(self.store.get(jobs[1]["id"])["state"], "submitted")
def test_continuity_size_is_validated(self):
with self.assertRaisesRegex(ValueError, "continuity"):
self.store.create({**self.spec, "continuity": "x" * 32001})
def test_storyboard_actions_have_no_fixed_count_cap(self):
requests = [{"index": i, "grid": 2, "prompt": "Same character, next action"} for i in range(12)]
self.assertEqual(actions.normalize_storyboard_actions(requests, set(range(12))), requests)
def test_replayed_generation_request_does_not_create_another_paid_job(self):
spec = {**self.spec, "request_key": "assistant-message:section-a"}
first = self.store.create(spec)
self.store.claim(first["id"])
replay = storyboards.StoryboardStore(self.store.path).create(spec)
self.assertEqual(replay["id"], first["id"])
self.assertEqual(replay["state"], "submitted")
self.assertEqual(len(self.store.list("project")), 1)
reroll = self.store.create({**spec, "request_key": "explicit-reroll"})
self.assertNotEqual(reroll["id"], first["id"])
other = self.store.create({**spec, "scheduler_id": "other"})
self.assertNotEqual(other["id"], first["id"])
with self.assertRaisesRegex(ValueError, "request key"):
self.store.create({**spec, "request_key": ""})
def test_moodboards_are_explicit_and_allowlisted(self):
Image.new("RGB", (32, 32)).save(self.root / "input" / "mood.png")
job = self.store.create({**self.spec, "moodboards": [{"filename": "mood.png", "type": "input", "role": "Color palette"}]})
graph = self.store.claim(job["id"])
self.assertEqual(graph["storyboard"]["inputs"]["model.images.image_1"], ["moodboard_1", 0])
self.assertNotIn("auth_token_comfy_org", json.dumps(graph))
with self.assertRaises(ValueError):
self.store.create({**self.spec, "grid": 4})
def test_extraction_exact_pixels_and_immutable_versions(self):
job = self.store.create(self.spec)
folder = self.root / "output" / "fl-storyboards" / job["id"]
folder.mkdir(parents=True)
sheet = Image.new("RGB", (100, 80), "red")
sheet.paste("blue", (50, 40, 100, 80))
sheet.save(folder / "sheet.png")
source = {"filename": "sheet.png", "subfolder": f"fl-storyboards/{job['id']}", "type": "output"}
result = storyboards.extract_panels(job, source)
self.assertEqual(len(result["assets"]), 4)
last = list(result["assets"].values())[-1]
with Image.open(references.reference_path(last)) as panel:
self.assertEqual(panel.size, (50, 40))
self.assertEqual(panel.getpixel((0, 0)), (0, 0, 255))
newer = storyboards.extract_panels({**job, "result": result}, source, [10, 10, 90, 70])
self.assertEqual(len(newer["versions"]), 2)
self.assertTrue(references.reference_path(last).is_file())
with self.assertRaises(ValueError):
storyboards.extract_panels(job, source, [-1, 0, 90, 70])
def test_agent_proposal_scope(self):
value = {"index": 2, "grid": 3, "prompt": "Nine clear chronological beats"}
self.assertEqual(actions.normalize_storyboard_actions([value], {2}), [value])
for values in ([value, value], [{**value, "index": 99}], [{**value, "grid": 4}]):
with self.assertRaises(ValueError):
actions.normalize_storyboard_actions(values, {2})
assignment = {"index": 2, "mode": "custom", "asset_ids": ["chosen"]}
self.assertEqual(actions.normalize_reference_assignments([assignment], {2}, ["chosen"]), [assignment])
with self.assertRaisesRegex(ValueError, "unavailable assets"):
actions.normalize_reference_assignments([assignment], {2}, [])
with self.assertRaisesRegex(ValueError, "unavailable or duplicate"):
actions.normalize_reference_assignments([assignment], {1}, ["chosen"])
def test_real_reference_library_loads_selected_pixels_only(self):
library_module = importlib.import_module("fl_reference_tests.FL_Prompt_Reference_Library")
Image.new("RGB", (64, 48), (255, 0, 0)).save(self.root / "input" / "selected.png")
schedule = {"reference_assets": {
"chosen": {"kind": "image", "filename": "selected.png"},
"unused": {"kind": "image", "filename": "missing.png"},
}, "sections": [{"references": {"mode": "custom", "asset_ids": ["chosen"]}}]}
library = library_module.FL_Prompt_Reference_Library()
value = library.load(schedule)[0]
self.assertEqual(list(value["assets"]), ["chosen"])
pixels = value["assets"]["chosen"]["value"]
self.assertEqual(tuple(pixels.shape), (1, 48, 64, 3))
self.assertEqual(pixels[0, 0, 0].tolist(), [1, 0, 0])
self.assertFalse(hasattr(library, "IS_CHANGED"), "connected schedules are unavailable during cache fingerprinting")
document = {"version": 1, "assets": schedule["reference_assets"],
"sections": [{"id": "shot", "mode": "custom", "asset_ids": ["chosen"]}]}
before = references.reference_file_fingerprint(json.dumps(document))
self.assertEqual(before, references.reference_file_fingerprint(json.dumps(document)))
Image.new("RGB", (65, 48), (0, 255, 0)).save(self.root / "input" / "selected.png")
self.assertNotEqual(before, references.reference_file_fingerprint(json.dumps(document)))
self.assertEqual(references.reference_file_fingerprint(""), ())
def test_real_audio_library_preserves_native_audio_shape(self):
library_module = importlib.import_module("fl_reference_tests.FL_Prompt_Reference_Library")
with wave.open(str(self.root / "input" / "sound.wav"), "wb") as audio:
audio.setnchannels(2)
audio.setsampwidth(2)
audio.setframerate(24000)
audio.writeframes(bytes(2400 * 2 * 2))
schedule = {"reference_assets": {"sound": {"kind": "audio", "filename": "sound.wav"}},
"sections": [{"references": {"mode": "custom", "asset_ids": ["sound"]}}]}
audio = library_module.FL_Prompt_Reference_Library().load(schedule)[0]["assets"]["sound"]["value"]
self.assertEqual(audio["sample_rate"], 24000)
self.assertEqual(tuple(audio["waveform"].shape), (1, 2, 2400))
def test_video_frame_rate_conversion_is_execution_cached(self):
video = torch.arange(30).reshape(30, 1, 1, 1)
library = {"version": 1, "assets": {"v": {"kind": "video", "value": video, "fps": 30.0}}}
sections = [{"references": {"mode": "custom", "asset_ids": ["v"]}}]
cache = {}
first = h3.resolve_shot_references(sections, library, None, None, None, None, cache)[1]["ref_video_0"]
second = h3.resolve_shot_references(sections, library, None, None, None, None, cache)[1]["ref_video_0"]
self.assertEqual(first.shape[0], 24)
self.assertIs(first, second)
self.assertEqual(len(cache), 1)
if __name__ == "__main__":
unittest.main()
+7 -1
View File
@@ -2,13 +2,17 @@ import importlib.util
import pathlib
import sys
import unittest
import types
from types import SimpleNamespace
from aiohttp import web
MODULE_PATH = pathlib.Path(__file__).parents[1] / "nodes" / "audio" / "prompt_writer_agent.py"
SPEC = importlib.util.spec_from_file_location("fl_prompt_writer_agent_tests", MODULE_PATH)
PACKAGE = types.ModuleType("fl_writer_agent_test_package")
PACKAGE.__path__ = [str(MODULE_PATH.parent)]
sys.modules[PACKAGE.__name__] = PACKAGE
SPEC = importlib.util.spec_from_file_location("fl_writer_agent_test_package.prompt_writer_agent", MODULE_PATH)
writer = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = writer
SPEC.loader.exec_module(writer)
@@ -262,6 +266,8 @@ class PromptWriterAgentTests(unittest.IsolatedAsyncioTestCase):
self.assertIn("## 7. Complete Example", system_prompt)
self.assertGreater(len(system_prompt), 23_000)
self.assertEqual([tool["function"]["name"] for tool in self.calls[0]["tools"]], [
"set_reference_assignments",
"generate_storyboards",
"get_prompt_boxes",
"plan_prompt_boxes",
"set_prompt_boxes",
+149
View File
@@ -0,0 +1,149 @@
import importlib.util
import json
from pathlib import Path
import sys
from types import ModuleType
import unittest
from unittest.mock import patch
import torch
import numpy as np
root = Path(__file__).parents[1] / "nodes"
for name, path in (("scan_edit_test", root), ("scan_edit_test.vfx", root / "vfx"), ("scan_edit_test.audio", root / "audio")):
package = ModuleType(name)
package.__path__ = [str(path)]
sys.modules[name] = package
spec = importlib.util.spec_from_file_location("scan_edit_test.vfx.FL_ScanAudioEdit", root / "vfx/FL_ScanAudioEdit.py")
edit = importlib.util.module_from_spec(spec)
spec.loader.exec_module(edit)
def envelope(values, fps=24):
return {"type": "fl_audio_envelope", "version": 1, "values": values, "fps": fps,
"duration": len(values) / fps, "total_frames": len(values)}
class ScanAudioEditTests(unittest.TestCase):
def test_random_order_changes_only_at_snare_onsets(self):
images = torch.rand(24,32,32,3)
quiet = envelope([0]*24)
snare = envelope([float(i in (3,4,12,13)) for i in range(24)])
observed = []
original_order = edit.order_windows
def capture(events,mode,ranks):
observed.append(tuple(ranks))
return original_order(events,mode,ranks)
with patch.object(edit,"order_windows",side_effect=capture):
edit.FL_ScanAudioEdit().render_mapped(images,images,images,quiet,snare,quiet,
"24",24,7,2,8,1.7,5,1,1,0,"audio_locked",window_order="random_on_snare")
self.assertEqual(len(observed),24)
for start,end in ((0,3),(3,12),(12,24)):
self.assertEqual(len(set(observed[start:end])),1)
self.assertNotEqual(observed[2],observed[3])
self.assertNotEqual(observed[11],observed[12])
def test_window_order_is_stable_and_priority_is_on_top(self):
events = [{"cursor":i,"effect":i} for i in range(3)]
self.assertEqual(edit.order_windows(events,"newest_on_top",[]),events)
for mode,effect in (("voxel_on_top",0),("edge_on_top",1),("depth_on_top",2)):
self.assertEqual(edit.order_windows(events,mode,[])[-1]["effect"],effect)
ranks = [2,0,1]
self.assertEqual([e["cursor"] for e in edit.order_windows(events,"random_on_snare",ranks)],[1,2,0])
self.assertEqual([e["cursor"] for e in edit.order_windows(events[1:],"random_on_snare",ranks)],[1,2])
def test_reveal_blends_and_fades(self):
for mode,value in (("normal",.5),("screen",.52),("add",.6)):
region = np.full((2,2,3),.2,np.float32)
edit.blend_reveal(region,np.full_like(region,.8),.5,mode)
np.testing.assert_allclose(region,value,atol=1e-7)
event = {"start":0,"end":25}
self.assertEqual(edit.reveal_fade(0,event,24,0,0),1)
self.assertEqual(edit.reveal_fade(2,event,24,.2,.2),0)
self.assertEqual(edit.reveal_fade(12,event,24,.2,.2),1)
self.assertEqual(edit.reveal_fade(24,event,24,.2,.2),0)
def test_cursor_events_follow_audio_not_cuts(self):
pulses = [float(i in (3, 21, 39)) for i in range(60)]
quiet = [0] * 60
events = edit.cursor_plan((pulses, quiet, quiet), 2, 99, 24)
self.assertEqual(events, edit.cursor_plan((pulses, quiet, quiet), 2, 99, 24))
self.assertEqual([e['start'] for e in events], [3, 21, 39])
self.assertEqual(edit.cursor_plan((quiet, quiet, quiet), 2, 99, 24), [])
self.assertNotEqual(events, edit.cursor_plan((pulses, quiet, quiet), 2, 100, 24))
for event in events:
self.assertTrue(all(.05 <= x <= .95 for x in event['target']))
self.assertEqual(edit.cursor_plan((pulses, pulses, pulses), 0, 99, 24), [])
def test_audio_locked_preserves_every_frame(self):
images = torch.rand(24, 32, 32, 3)
beat = envelope([float(i % 7 == 0) for i in range(24)])
_, before, _, report = edit.FL_ScanAudioEdit().render(images, images, images, beat, beat, beat,
"12,12", 24, 1, 2, 8, 1.7, 2, 1, 1, .8, timing_mode="audio_locked")
torch.testing.assert_close(before, images)
report = json.loads(report)
self.assertEqual(report['source_indices'], list(range(24)))
self.assertTrue(any(s['start_frame'] == 12 for s in report['segments']))
self.assertTrue(any(s['edit_type'] == 'camera_cut' for s in report['segments']))
def test_plan_is_repeatable_in_bounds_and_onset_driven(self):
kick = [float(frame in (5, 12, 20, 28)) for frame in range(36)]
args = ([12, 12, 12, 12], kick, [0] * 36, 4, 10, 1.7, 73)
indices, segments = edit.edit_plan(*args)
self.assertEqual((indices, segments), edit.edit_plan(*args))
self.assertEqual(len(indices), 36)
self.assertTrue(all(0 <= index < 48 for index in indices))
self.assertEqual({s['shot'] for s in segments[:4]}, {1, 2, 3, 4})
self.assertEqual(segments[1]['start_frame'], 5)
self.assertEqual(segments[1]['trigger'], 'audio_onset')
self.assertEqual(segments[4]['edit_type'], 'jump_cut')
self.assertEqual(segments[4]['shot'], segments[3]['shot'])
for segment in segments:
bank = (segment['shot'] - 1) * 12
self.assertTrue(all(bank <= i < bank + 12 for i in indices[segment['start_frame']:segment['end_frame']]))
def test_invalid_cut_bounds(self):
with self.assertRaisesRegex(ValueError, "maximum cut length"):
edit.edit_plan([12], [0] * 12, [0] * 12, 10, 5, 1, 0)
def test_aligned_outputs_masks_and_no_mutation(self):
source = torch.arange(24).reshape(24, 1, 1, 1).expand(24, 64, 64, 3).float() / 24
scan = torch.ones_like(source) * 0.2
normal = torch.zeros_like(source)
normal[:, :, :, 1] = 1
original = source.clone()
beats = envelope([float(i % 7 == 0) for i in range(24)])
result, before, mask, report = edit.FL_ScanAudioEdit().render(source, scan, normal, beats, beats, beats,
"12,12", 24, 73, 6, 12, 1.7, 2, 1.2, 1, 0.8)
self.assertEqual(result.shape, source.shape)
self.assertEqual(tuple(mask.shape), (24, 64, 64))
self.assertTrue((mask >= 0).all() and (mask <= 1).all())
self.assertGreater(float(mask.sum()), 0)
self.assertTrue(torch.isfinite(result).all())
self.assertTrue((result >= 0).all() and (result <= 1).all())
indices = json.loads(report)['source_indices']
torch.testing.assert_close(before, source[indices])
torch.testing.assert_close(source, original)
self.assertTrue((normal[:, :, :, 1] == 1).all())
def test_wrong_envelope_fps_and_bank_length_fail(self):
images = torch.zeros(4, 32, 32, 3)
beat = envelope([0] * 4)
args = [images, images, images, beat, beat, envelope([0] * 4, 30), "4", 24, 1, 2, 4, 1, 1, 1, 1, 1]
with self.assertRaisesRegex(ValueError, "match the edit FPS"):
edit.FL_ScanAudioEdit().render(*args)
args[5], args[6] = beat, "5"
with self.assertRaisesRegex(ValueError, "add up"):
edit.FL_ScanAudioEdit().render(*args)
def test_zero_reveal_strength_outputs_empty_mask(self):
images = torch.zeros(8, 64, 64, 3)
beat = envelope([0] * 8)
_, _, mask, _ = edit.FL_ScanAudioEdit().render(images, images, images, beat, beat, beat,
"8", 24, 1, 2, 8, 1, 2, 1, 0, 0)
self.assertEqual(float(mask.sum()), 0)
if __name__ == "__main__":
unittest.main()
+76
View File
@@ -0,0 +1,76 @@
import importlib
from pathlib import Path
import sys
from types import ModuleType
import unittest
import numpy as np
root=Path(__file__).parents[1]/"nodes"
for name,path in (("modulation_test",root),("modulation_test.vfx",root/"vfx"),("modulation_test.audio",root/"audio")):
package=ModuleType(name);package.__path__=[str(path)];sys.modules[name]=package
fx=importlib.import_module("modulation_test.vfx.FL_InteractiveScanFX")
mod=importlib.import_module("modulation_test.vfx.scan_modulation")
scan=importlib.import_module("modulation_test.vfx.FL_StreetScan")
edit=importlib.import_module("modulation_test.vfx.FL_ScanAudioEdit")
class MappingTests(unittest.TestCase):
def curves(self,rows):
settings=fx.DEFAULTS|{"relief":.65,"animation":.18,"speed":.7,"motion_strength":.6,"audio_mappings":rows}
return mod.compile_mappings(settings,[{"values":[0,1,0,1,0,1,0,1]}]*3,24,[4,4])
def row(self,**changes):
return {"source":0,"target":"relief","minimum":.5,"maximum":1,"start_frame":2,"end_frame":6,"smoothing":0}|changes
def test_mapping_is_absolute_and_only_active_in_range(self):
values,_=self.curves([self.row()])
self.assertEqual(values["relief"],[.65,.65,.5,1,.5,1,.65,.65])
values,_=self.curves([self.row(invert=True)])
self.assertEqual(values["relief"][2:6],[1,.5,1,.5])
def test_smoothing_resets_at_shot_boundary(self):
values,_=self.curves([self.row(start_frame=0,end_frame=None,smoothing=.2)])
self.assertGreater(values["relief"][1],.5)
self.assertLess(values["relief"][1],1)
self.assertEqual(values["relief"][4],.5)
def test_bad_ranges_and_overlaps_fail(self):
for rows in ([self.row(end_frame=9)],[self.row(source=3)],[self.row(target="fps")],
[self.row(),self.row(start_frame=5,end_frame=8)], [self.row(minimum=float("nan"))]):
with self.assertRaises(ValueError):self.curves(rows)
self.curves([self.row(end_frame=4),self.row(start_frame=4)])
def test_disabled_row_does_not_override(self):
values,_=self.curves([self.row(enabled=False)])
self.assertEqual(values["relief"],[.65]*8)
def test_zero_weight_total_fails(self):
with self.assertRaisesRegex(ValueError,"positive total"):
self.curves([self.row(target="voxel_weight",minimum=0,maximum=0,start_frame=0,end_frame=None),
self.row(target="edge_weight",minimum=0,maximum=0,start_frame=0,end_frame=None)])
def test_depth_selection_is_seeded_and_latched(self):
values,_=self.curves([])
values["depth_weight"]=[100]*8;values["voxel_weight"]=[0]*8;values["edge_weight"]=[0]*8
def events():return edit.cursor_plan(([1,0,0,0,1,0,0,0],)*3,3,10,24)
a=edit.assign_reveal_layers(events(),values,73);b=edit.assign_reveal_layers(events(),values,73)
self.assertTrue(a);self.assertEqual(a,b);self.assertTrue(all(e["effect"]==2 for e in a))
def test_native_camera_anchors_steady_depth(self):
d=np.full((64,64),.6,np.float32)
x,y,z=scan.project_parallax(d,0,0,1,1,1,.1,-.1,.2,.6)
yy,xx=np.mgrid[:64,:64]
np.testing.assert_allclose(x,xx,atol=1e-5);np.testing.assert_allclose(y,yy,atol=1e-5)
moved=scan.project_parallax(np.full_like(d,.1),0,0,1,1,1,.1,-.1,.2,.6)
self.assertGreater(float(np.abs(moved[0]-x).max()),.5)
self.assertTrue(np.isfinite(moved[2]).all())
def test_native_camera_matches_old_at_neutral_settings(self):
d=np.random.default_rng(1).random((48,64)).astype(np.float32)
old=scan.project_depth(d,.35,5.5,1,.74)
new=scan.project_parallax(d,.35,5.5,1,.74,0,0,0,0,.5)
for a,b in zip(old,new):np.testing.assert_allclose(a,b,atol=1e-5)
if __name__=="__main__":unittest.main()
+58
View File
@@ -0,0 +1,58 @@
import importlib
from pathlib import Path
import sys
from types import ModuleType, SimpleNamespace
import unittest
from unittest.mock import Mock, patch
import torch
package = ModuleType("fl_seg_krea_test")
package.__path__ = [str(Path(__file__).parents[1] / "nodes")]
sys.modules[package.__name__] = package
m = importlib.import_module("fl_seg_krea_test.ksamplers.FL_KsamplerSEG_Krea")
common = importlib.import_module("fl_seg_krea_test.ksamplers.FL_KsamplerSEG_common")
def model():
objects = {"latent_format": SimpleNamespace(spacial_downscale_ratio=8),
"model_sampling": SimpleNamespace(percent_to_sigma=lambda p: 1 - p)}
result = SimpleNamespace(model=Mock(spec=m.comfy.model_base.Krea2), model_options={},
is_dynamic=lambda: False, get_model_object=objects.__getitem__)
result.clone = model
result.set_model_sampler_calc_cond_batch_function = lambda fn: result.model_options.update(sampler_calc_cond_batch_function=fn)
return result
class SegKreaTests(unittest.TestCase):
def test_each_reference_matches_sampler_crop_without_changing_source(self):
source = torch.rand(1, 64, 96, 3)
original = source.clone()
masks = torch.ones(2, 64, 96)
regions = common.make_regions_dict(masks, masks, masks, [(1, 3, 31, 47), (25, 41, 64, 96)], (64, 96), 8, 1)
clip = Mock(tokenizer=Mock(spec=m.Krea2Tokenizer))
baseline = [[torch.ones(1, 2, 3), {}]]
clip.encode_from_tokens_scheduled.return_value = baseline
for strength in (0, .5, 1):
clip.reset_mock()
with patch.object(m, "encode_reference", return_value=[[torch.zeros(1, 2, 3), {}]]) as encode:
original_model = model()
patched, encoded, cond = m.FL_KsamplerSEG_Krea().encode(original_model, clip, regions, source, "preserve", strength)
self.assertEqual(clip.encode_from_tokens_scheduled.call_count, 1)
self.assertEqual(encode.call_count, 2 if strength else 0)
self.assertIs(cond, baseline)
self.assertEqual(original_model.model_options, {})
self.assertIs(patched.model_options["sampler_calc_cond_batch_function"], m.reference_cond_batch)
self.assertIsNone(regions["conditioning_per_region"])
for i, call in enumerate(encode.call_args_list):
by0, bx0, by1, bx1 = common.latent_bbox_from_image_bbox(regions["padded_bboxes"][i], 8, 8, 12)
torch.testing.assert_close(call.args[2]["image"], source[:, by0 * 8:by1 * 8, bx0 * 8:bx1 * 8])
self.assertEqual(encoded["conditioning_per_region"][i][0][-1][1]["fl_krea_reference"][1], strength)
torch.testing.assert_close(source, original)
with self.assertRaisesRegex(ValueError, "same size"):
m.FL_KsamplerSEG_Krea().encode(model(), clip, regions, source[:, :32], "preserve")
if __name__ == "__main__":
unittest.main()
+53
View File
@@ -0,0 +1,53 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
import vm from "node:vm";
const source = await readFile(
new URL("../web/nodes/ksamplers/FL_KsamplerSEG_Regions.js", import.meta.url),
"utf8",
);
test("SEG Regions preview uses stable string node keys", () => {
assert.match(source, /const nodeKey = \(value\) => String\(value\);/);
assert.match(source, /INSTANCES\.set\(nodeKey\(node\.id\), inst\);/);
assert.match(source, /INSTANCES\.get\(nodeKey\(detail\.node\)\)/);
assert.doesNotMatch(source, /parseInt\(detail\.node/);
});
test("old workflows retain factors and new workflows expose pixel controls", () => {
let extension;
vm.runInNewContext(source.replace(/^import .*;$/gm, ""), {
app: { registerExtension: value => { extension = value; } },
api: { addEventListener() {} },
document: { createElement: () => ({ style: {} }) },
setTimeout() {},
});
const names = ["num_regions", "relaxation_iterations", "region_overlap_factor", "edge_softness", "context_padding_factor", "safe_zone_feather_px", "downscale_ratio", "seed", "control_after_generate", "show_preview", "preview_mode", "margin_mode", "overlap_width_px", "feather_width_px", "context_padding_px"];
const node = {
constructor: { comfyClass: "FL_KsamplerSEG_Regions" },
widgets: names.map(name => ({ name, type: "number", options: {}, value: name === "margin_mode" ? "pixels" : 0 })),
size: [380, 560], setSize() {}, setDirtyCanvas() {},
addDOMWidget: () => ({}),
};
extension.nodeCreated(node);
const widget = name => node.widgets.find(w => w.name === name);
assert.equal(widget("region_overlap_factor").hidden, true);
assert.notEqual(widget("overlap_width_px").hidden, true);
node.onConfigure({ widgets_values: [4, 5, .15, .1, .2, 8, 8, 1, "fixed", true, "overlay", ""] });
assert.equal(widget("margin_mode").value, "legacy");
assert.notEqual(widget("region_overlap_factor").hidden, true);
assert.equal(widget("overlap_width_px").hidden, true);
widget("margin_mode").value = "pixels";
widget("margin_mode").callback();
assert.equal(widget("region_overlap_factor").hidden, true);
assert.equal(widget("overlap_width_px").type, "number");
widget("feather_width_px").value = 32;
widget("overlap_width_px").value = 16;
widget("overlap_width_px").callback();
assert.equal(widget("feather_width_px").value, 8);
assert.equal(widget("feather_width_px").options.max, 8);
widget("overlap_width_px").value = 0;
widget("overlap_width_px").callback();
assert.equal(widget("feather_width_px").value, 0);
});
+215
View File
@@ -0,0 +1,215 @@
import importlib.util
from pathlib import Path
import sys
from types import ModuleType, SimpleNamespace
import unittest
from unittest.mock import patch
import numpy as np
import torch
root = Path(__file__).parents[1] / "nodes/ksamplers"
package = ModuleType("fl_seg_test")
package.__path__ = [str(root)]
sys.modules[package.__name__] = package
def load(name):
spec = importlib.util.spec_from_file_location(f"fl_seg_test.{name}", root / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
m = load("FL_KsamplerSEG")
r = load("FL_KsamplerSEG_Regions")
def model(ratio=8):
objects = {"latent_format": SimpleNamespace(spacial_downscale_ratio=ratio, latent_dimensions=2, latent_channels=16),
"model_sampling": object(), "process_latent_in": lambda x: x * 2 - 1}
return SimpleNamespace(load_device=torch.device("cpu"), model_options={}, get_model_object=objects.__getitem__)
class SegSamplerTests(unittest.TestCase):
def advanced_args(self):
masks = torch.zeros(2, 128, 256)
masks[0, :, :128] = 1
masks[1, :, 128:] = 1
regions = r.make_regions_dict(masks, masks, masks, [(0, 0, 128, 128), (0, 128, 128, 256)], (128, 256), 8, 1)
regions['conditioning_per_region'] = [('first', 'negative'), ('second', 'negative')]
return dict(model=model(), regions=regions, latent_image={'samples': torch.ones(1, 16, 16, 32), 'metadata': 'keep'},
positive='default', negative='negative', noise_seed=10, steps=8, cfg=1,
sampler_name='euler', scheduler='simple', start_at_step=2, end_at_step=5,
add_noise='enable', return_with_leftover_noise='enable', cascade_start_corner='top_left')
def test_advanced_uses_native_sigma_window_noise_controls_and_progress(self):
for add_noise in ('enable', 'disable'):
for leftover in ('enable', 'disable'):
args = self.advanced_args()
args.update(add_noise=add_noise, return_with_leftover_noise=leftover)
calls, progress = [], []
def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, options, **kwargs):
calls.append((noise.clone(), positive, sigmas.clone(), kwargs['seed']))
latent = kwargs['latent_image']
for step in range(len(sigmas)):
kwargs['callback'](step, latent, latent, len(sigmas) - 1)
return latent + 1
with patch.object(m.comfy.samplers, 'calculate_sigmas', side_effect=lambda sampling, scheduler, steps: torch.linspace(1, 0, steps + 1)), \
patch.object(m.comfy.samplers, 'sample', side_effect=sample), \
patch.object(m.latent_preview, 'prepare_callback', return_value=lambda step, x0, x, total: progress.append((step, total))) as prepare:
result = m.FL_KsamplerSEGAdvanced().sample(**args)[0]
expected = torch.tensor([.75, .625, .5, .375 if leftover == 'enable' else 0])
self.assertEqual([c[1] for c in calls], ['first', 'second'])
self.assertEqual([c[3] for c in calls], [10, 11])
for noise, _, sigmas, _ in calls:
torch.testing.assert_close(sigmas, expected)
self.assertEqual(bool(noise.count_nonzero()), add_noise == 'enable')
prepare.assert_called_once_with(args['model'], 6)
self.assertEqual(progress, [(i, 6) for i in (0, 1, 2, 2, 3, 4, 5, 5)])
torch.testing.assert_close(result['samples'], torch.full_like(result['samples'], 2))
torch.testing.assert_close(args['latent_image']['samples'], torch.ones_like(result['samples']))
self.assertEqual(result['metadata'], 'keep')
def test_advanced_empty_windows_do_not_sample_or_add_noise(self):
for start, end in ((5, 5), (6, 3), (8, 10000), (0, 0)):
args = self.advanced_args()
args.update(start_at_step=start, end_at_step=end)
with patch.object(m.comfy.sample, 'prepare_noise') as noise, patch.object(m.comfy.sample, 'sample') as sample:
result = m.FL_KsamplerSEGAdvanced().sample(**args)
self.assertIs(result[0], args['latent_image'])
noise.assert_not_called()
sample.assert_not_called()
def test_advanced_window_matches_regular_denoise_schedule_and_noise(self):
args = self.advanced_args()
calls = []
def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, options, **kwargs):
calls.append((noise.clone(), sigmas.clone()))
return kwargs['latent_image']
with patch.object(m.comfy.samplers, 'calculate_sigmas', side_effect=lambda sampling, scheduler, steps: torch.linspace(1, 0, steps + 1)), \
patch.object(m.comfy.samplers, 'sample', side_effect=sample), \
patch.object(m.latent_preview, 'prepare_callback', return_value=lambda *args: None):
regular = {k: args[k] for k in ('model', 'regions', 'latent_image', 'positive', 'negative', 'cfg', 'sampler_name', 'scheduler', 'cascade_start_corner')}
m.FL_KsamplerSEG().sample(**regular, seed=10, steps=4, denoise=.2)
args.update(steps=20, start_at_step=16, end_at_step=10000, return_with_leftover_noise='disable')
m.FL_KsamplerSEGAdvanced().sample(**args)
self.assertEqual(len(calls), 4)
for regular, advanced in zip(calls[:2], calls[2:]):
torch.testing.assert_close(regular[0], advanced[0])
torch.testing.assert_close(regular[1], advanced[1])
self.assertEqual(len(advanced[1]), 5)
def test_advanced_schema_has_step_controls_without_denoise(self):
required = m.FL_KsamplerSEGAdvanced.INPUT_TYPES()['required']
self.assertNotIn('denoise', required)
self.assertNotIn('seed', required)
for name in ('add_noise', 'noise_seed', 'start_at_step', 'end_at_step', 'return_with_leftover_noise'):
self.assertIn(name, required)
self.assertIn('denoise', m.FL_KsamplerSEG.INPUT_TYPES()['required'])
def test_pixel_overlap_has_bounded_support_and_full_coverage(self):
hard = torch.zeros(2, 8, 20)
hard[0, :, :10] = 1
hard[1, :, 10:] = 1
for pixel in (1, 8):
for feather in (0, pixel, 2 * pixel):
masks = r.FL_KsamplerSEG_Regions._pixel_masks(hard, 2 * pixel, feather, pixel, pixel)
self.assertTrue(torch.all(masks.sum(0) >= 1))
self.assertTrue(torch.all(masks[0, :, 12:] == 0))
self.assertTrue(torch.all(masks[1, :, :8] == 0))
self.assertTrue(torch.all(masks[:, :, 8:12] > 0))
torch.testing.assert_close(r.FL_KsamplerSEG_Regions._pixel_masks(hard, 0, 0, 8, 8), hard)
def test_feather_does_not_grow_crop_and_context_does_not_grow_mask(self):
node = r.FL_KsamplerSEG_Regions()
args = dict(num_regions=4, relaxation_iterations=2, region_overlap_factor=.15,
edge_softness=.1, context_padding_factor=.2, safe_zone_feather_px=0,
downscale_ratio=8, seed=12, show_preview=False, preview_mode="sampler_crops",
image=torch.zeros(1, 256, 256, 3), margin_mode="pixels", overlap_width_px=32)
a = node.build(**args, feather_width_px=0, context_padding_px=0)[0]
b = node.build(**args, feather_width_px=16, context_padding_px=0)[0]
c = node.build(**args, feather_width_px=16, context_padding_px=24)[0]
self.assertEqual(a["padded_bboxes"], b["padded_bboxes"])
self.assertNotEqual(b["padded_bboxes"], c["padded_bboxes"])
torch.testing.assert_close(b["write_masks"], c["write_masks"])
zero = node.build(**dict(args, overlap_width_px=0), feather_width_px=0, context_padding_px=0)[0]
self.assertTrue(torch.all((zero["write_masks"] > 0).sum(0) == 1))
with self.assertRaisesRegex(ValueError, "half the overlap"):
node.build(**args, feather_width_px=24)
def test_wrong_image_size_fails_instead_of_guessing_downscale(self):
with self.assertRaisesRegex(ValueError, "same resized image"):
m.FL_KsamplerSEG._resolve_downscale(model(), {}, 512, 412, 1152, 928)
self.assertEqual(m.FL_KsamplerSEG._resolve_downscale(model(), {}, 512, 412, 4096, 3296), 8)
def test_model_ratio_and_vae_rounding(self):
self.assertEqual(m.FL_KsamplerSEG._resolve_downscale(model(32), {}, 32, 24, 1025, 769), 32)
self.assertEqual(m.FL_KsamplerSEG._resolve_downscale(model(32), {}, 33, 25, 1025, 769), 32)
with self.assertRaises(ValueError):
m.FL_KsamplerSEG._resolve_downscale(model(32), {}, 34, 25, 1025, 769)
def test_full_canvas_preview_progress_and_latent_are_independent(self):
for shape in ((1, 16, 24, 32), (1, 16, 1, 24, 32)):
with self.subTest(shape=shape):
source = torch.ones(shape)
original = source.clone()
spec = dict(latent_bbox=(4, 8, 20, 24), seed=3, region_index=2,
write_lat=torch.ones(16, 16), comp_lat=torch.full((16, 16), .5),
pos_cond=object(), neg_cond=object())
frames = []
def preview(step, x0, x, total):
frames.append((step, x0.clone(), total))
def sample(model, noise, steps, cfg, sampler, scheduler, positive, negative, latent, **kwargs):
self.assertIs(positive, spec["pos_cond"])
self.assertIs(negative, spec["neg_cond"])
for step in range(steps):
kwargs["callback"](step, torch.full_like(latent, 5 + step), latent, steps)
return torch.full_like(latent, 9)
with patch.object(m.comfy.sample, "sample", side_effect=sample):
result = m.FL_KsamplerSEG()._sample_one_region_full(
model=model(), source_latent=source, spec=spec, steps=2, cfg=1,
sampler_name="euler", scheduler="simple", denoise=.3,
preview_callback=preview, step_offset=4, total_steps=8)
self.assertEqual([(i, total) for i, _, total in frames], [(4, 8), (5, 8)])
for i, (_, frame, _) in enumerate(frames):
expected = torch.ones_like(source)
expected[..., 4:20, 8:24] = 3 + i * .5
torch.testing.assert_close(frame, expected)
torch.testing.assert_close(source, original)
torch.testing.assert_close(result, torch.full_like(result, 9))
def test_raw_regions_keep_reference_conditioning(self):
marker = ("reference_0", .42, (1, 1, 0, 0))
positive = [[torch.zeros(1), {"fl_krea_reference": marker}]]
negative = [[torch.zeros(1), {}]]
regions = dict(padded_bboxes=[(0, 0, 128, 128)], write_masks=torch.ones(1, 128, 128),
composite_masks=torch.ones(1, 128, 128))
spec = m.FL_KsamplerSEG()._build_region_spec(
regions=regions, region_index=0, downscale=8, latent_h=16, latent_w=16,
device="cpu", per_region_cond=None, cond_pos_default=positive,
cond_neg_default=negative, base_seed=3)
self.assertIs(spec["pos_cond"], positive)
self.assertEqual(spec["pos_cond"][0][1]["fl_krea_reference"], marker)
def test_crop_preview_does_not_change_masks(self):
image = np.full((64, 64, 3), 160, dtype=np.uint8)
masks = torch.zeros(2, 64, 64)
masks[0, :, :32] = 1
masks[1, :, 32:] = 1
original = masks.clone()
preview = r.FL_KsamplerSEG_Regions._build_crop_viz(image, masks, [(0, 0, 64, 48), (0, 16, 64, 64)])
self.assertEqual(preview.shape, (350, 640, 3))
torch.testing.assert_close(masks, original)
if __name__ == "__main__":
unittest.main()
+141
View File
@@ -0,0 +1,141 @@
import importlib.util
from pathlib import Path
from types import SimpleNamespace
import unittest
import numpy as np
import cv2
import math
import torch
spec = importlib.util.spec_from_file_location("street_scan_test", Path(__file__).parents[1] / "nodes/vfx/FL_StreetScan.py")
scan = importlib.util.module_from_spec(spec)
spec.loader.exec_module(scan)
class StreetScanTests(unittest.TestCase):
def test_default_stack_is_pixel_identical_to_original(self):
matte = np.zeros((96,128),np.uint8)
matte[16:75,20:98] = 255
layers = np.random.default_rng(5).random((96,128,6),dtype=np.float32)
for phase in (0,.3,1):
filled,hull = scan.fill_projected_gaps(layers.copy(),matte.copy())
expected = np.zeros((96,128,3),np.float32)
step = max(2,round(128*(.009+.004)))
for level in (4,3,2,1):
shift = np.float32([[1,0,level*step],[0,1,level*step*(.65+.2*math.sin(phase*math.tau))]])
plate = cv2.warpAffine(hull,shift,(128,96),flags=cv2.INTER_NEAREST)
expected[plate>0] = (.035,.02,.9) if level%2==0 else (.04,.04,.055)
contours,_ = cv2.findContours(plate,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(expected,contours,-1,(.85,.9,1),1,cv2.LINE_8)
expected[hull>0] = filled[:,:,:3][hull>0]
contours,_ = cv2.findContours(hull,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(expected,contours,-1,(1,1,1),1,cv2.LINE_8)
actual = scan.digital_layers(layers.copy(),matte.copy(),phase,1)[2]
np.testing.assert_array_equal(actual,expected)
def test_stack_controls_preserve_foreground_and_disable_cleanly(self):
matte = np.zeros((96,128),np.uint8)
matte[24:64,36:80] = 255
layers = np.full((96,128,6),.3,np.float32)
base = scan.digital_layers(layers.copy(),matte.copy(),.2,1)[2]
off = scan.digital_layers(layers.copy(),matte.copy(),.2,1,count=0)[2]
transparent = scan.digital_layers(layers.copy(),matte.copy(),.2,1,opacity=0)[2]
np.testing.assert_array_equal(off,transparent)
for options in ({"count":8},{"spacing":3},{"x":-1},{"y":-1},{"rotation":8},{"opacity":.4},{"palette":"cyan"}):
result = scan.digital_layers(layers.copy(),matte.copy(),.2,1,**options)[2]
self.assertFalse(np.array_equal(result,base),options)
np.testing.assert_array_equal(result[26:62,38:78],base[26:62,38:78])
self.assertTrue(np.isfinite(result).all())
def test_digital_mode_does_not_use_eroded_mask(self):
images = torch.rand(2, 32, 32, 3)
depth = torch.ones_like(images) * .5
masks = torch.zeros(2, 32, 32)
tracks = {"width": 32, "height": 32, "frames": [[], []]}
args = (images, depth, depth, masks, tracks, 24, 41, 5, 1, .74)
first = scan.FL_StreetScanComposite().render(*args, 0, 0, 0, 0, edge_style='digital_layers')
second = scan.FL_StreetScanComposite().render(*args, 1, 0, 0, 1, edge_style='digital_layers')
for a, b in zip(first, second):
torch.testing.assert_close(a, b)
def test_digital_layers_fill_projection_gaps_and_keep_normals_aligned(self):
layers = np.zeros((32, 32, 6), np.float32)
layers[4:25, 4:25] = [.2, .3, .4, .6, .7, .8]
matte = np.zeros((32, 32), np.uint8)
matte[4:25, 4:25] = 255
matte[10:12, 10:12] = 0
layers[10:12, 10:12] = 0
result, filled, composite = scan.digital_layers(layers, matte, .2, 1)
np.testing.assert_allclose(result[10, 10], [.2, .3, .4, .6, .7, .8])
self.assertTrue((filled[4:25, 4:25] == 255).all())
self.assertTrue(np.isfinite(composite).all())
self.assertTrue((composite[filled == 0] > 0).any())
def test_identity_projection_preserves_pixels(self):
rgb = np.random.default_rng(10).random((32, 32, 3), dtype=np.float32)
depth = np.ones((32, 32), np.float32) * 0.5
px, py, z = scan.project_depth(depth, 0, 0, 1, 1)
result, matte = scan.splat(rgb, np.ones((32, 32), bool), px, py, z, (0, 0, 0))
np.testing.assert_array_equal(result, rgb)
self.assertTrue((matte == 255).all())
def test_z_buffer_keeps_nearest_surface(self):
rgb = np.array([[[1, 0, 0], [0, 1, 0]]], np.float32)
result, _ = scan.splat(rgb, np.ones((1, 2), bool), np.zeros((1, 2)), np.zeros((1, 2)), np.array([[2, 1]]), (0, 0, 0))
np.testing.assert_array_equal(result[0, 0], [0, 1, 0])
def test_fragment_is_deterministic_and_preserves_subject(self):
depth = np.ones((64, 64), np.float32) * 0.5
subject = np.zeros_like(depth)
subject[:5, :5] = 1
mask = scan.scene_fragment(depth, subject, 41, 0.5)
np.testing.assert_array_equal(mask, scan.scene_fragment(depth, subject, 41, 0.5))
self.assertTrue(mask[:5, :5].all())
self.assertFalse(mask.all())
def test_detector_produces_real_masks_and_stable_ids(self):
segment = SimpleNamespace(bbox=(4, 5, 20, 25), crop_region=(4, 5, 20, 25),
cropped_mask=np.ones((20, 16), np.float32), confidence=0.9)
detector = SimpleNamespace(detect=lambda *args, **kwargs: ((32, 32), [segment]))
masks, tracks = scan.FL_ScanVideoDetections().detect(torch.zeros(2, 32, 32, 3), detector, 0.35)
self.assertEqual(tuple(masks.shape), (2, 32, 32))
self.assertEqual(tracks["frames"][0][0]["id"], tracks["frames"][1][0]["id"])
self.assertEqual(float(masks[0].sum()), 320)
def test_composite_is_bounded_repeatable_and_does_not_modify_inputs(self):
images = torch.rand(3, 64, 64, 3)
original = images.clone()
depth = torch.ones_like(images) * 0.5
masks = torch.zeros(3, 64, 64)
tracks = {"width": 64, "height": 64, "frames": [[], [], []]}
args = (images, depth, depth, masks, tracks, 24, 41, 8, 1.4, 0.88, 0.4, 0.8, 0.8, 0.4)
first, matte, normals = scan.FL_StreetScanComposite().render(*args)
second, _, _ = scan.FL_StreetScanComposite().render(*args)
torch.testing.assert_close(first, second)
torch.testing.assert_close(images, original)
self.assertTrue(torch.isfinite(first).all())
self.assertTrue((first >= 0).all() and (first <= 1).all())
self.assertEqual(tuple(matte.shape), (3, 64, 64))
self.assertEqual(normals.shape, images.shape)
self.assertTrue(torch.isfinite(normals).all())
def test_pose_overlay_uses_canvas_coordinates_and_rejects_wrong_batch(self):
images = torch.zeros(1, 64, 64, 3)
depth = torch.ones_like(images) * 0.5
masks = torch.zeros(1, 64, 64)
tracks = {"width": 64, "height": 64, "frames": [[]]}
args = (images, depth, depth, masks, tracks, 24, 20, 0, 1, 1, 0.4, 0, 0, 0)
points = [[16 + index * 4, 64, 1] for index in range(18)]
pose = [{"canvas_width": 128, "canvas_height": 128,
"people": [{"pose_keypoints_2d": [v for point in points for v in point]}]}]
plain, _, _ = scan.FL_StreetScanComposite().render(*args)
overlay, _, _ = scan.FL_StreetScanComposite().render(*args, pose_keypoints=pose, pose_opacity=1)
self.assertGreater(float((overlay[0, 30:35] - plain[0, 30:35]).abs().sum()), 0)
with self.assertRaisesRegex(ValueError, "one DWPose keypoint frame"):
scan.FL_StreetScanComposite().render(*args, pose_keypoints=[])
if __name__ == "__main__":
unittest.main()
+59
View File
@@ -0,0 +1,59 @@
import importlib.util
from pathlib import Path
import unittest
import torch
spec = importlib.util.spec_from_file_location('voxel_test', Path(__file__).parents[1] / 'nodes/vfx/FL_VoxelNormalRelief.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
class VoxelNormalTests(unittest.TestCase):
def test_output_is_aligned_bounded_deterministic_and_input_unchanged(self):
normals = torch.rand(3, 47, 61, 3)
depth = torch.rand_like(normals)
original = normals.clone()
args = (normals, depth, 8, .65, .2, .7, 24, 41)
a = module.FL_VoxelNormalRelief().render(*args)[0]
b = module.FL_VoxelNormalRelief().render(*args)[0]
self.assertEqual(a.shape, normals.shape)
torch.testing.assert_close(a, b)
torch.testing.assert_close(normals, original)
self.assertTrue(torch.isfinite(a).all() and (a >= 0).all() and (a <= 1).all())
def test_animation_and_depth_change_relief(self):
normals = torch.ones(12, 48, 48, 3) * .7
depth = torch.ones_like(normals) * .5
node = module.FL_VoxelNormalRelief()
moving = node.render(normals, depth, 12, 1, .4, 1, 24, 41)[0]
still = node.render(normals, depth, 12, 1, 0, 1, 24, 41)[0]
torch.testing.assert_close(still[0], still[-1])
self.assertFalse(torch.equal(moving[0], moving[-1]))
flat = node.render(normals, torch.zeros_like(depth), 12, 1, 0, 1, 24, 41)[0]
self.assertFalse(torch.equal(flat, still))
def test_mismatched_frames_fail(self):
with self.assertRaisesRegex(ValueError, 'aligned'):
module.FL_VoxelNormalRelief().render(torch.zeros(2, 32, 32, 3), torch.zeros(1, 32, 32, 3),
8, 1, .2, 1, 24, 0)
def test_flat_surface_keeps_visible_cube_sides(self):
normals = torch.ones(1, 48, 48, 3) * .7
depth = torch.ones_like(normals) * .5
image = module.FL_VoxelNormalRelief().render(normals, depth, 12, .65, 0, .7, 24, 41)[0]
for shade in (.48, .68):
self.assertTrue(torch.isclose(image[..., 0], torch.tensor(.7 * shade)).any())
def test_speed_zero_holds_accumulated_phase(self):
normals=torch.ones(12,48,48,3)*.7
depth=torch.ones_like(normals)*.5
values={"relief":[1]*12,"animation":[.8]*12,"speed":[1]*4+[0]*8}
output=module.FL_VoxelNormalRelief().render_animated(normals,depth,12,1,.8,1,24,41,values)[0]
torch.testing.assert_close(output[4],output[-1])
self.assertFalse(torch.equal(output[0],output[4]))
if __name__ == '__main__':
unittest.main()
@@ -80,6 +80,7 @@ app.registerExtension({
analysisSource: findWidget(node, "analysis_source"),
beatGridDensity: findWidget(node, "beat_grid_density"),
renderGroups: findWidget(node, "render_groups"),
referenceSchedule: findWidget(node, "reference_schedule"),
analysisCacheKey: findWidget(node, "analysis_cache_key"),
envelopeLayers: findWidget(node, "envelope_layers"),
};
+152
View File
@@ -0,0 +1,152 @@
import { PromptWriterClient } from "./audio_prompt_writer_client.js";
import { ensureReferenceWiring } from "./audio_prompt_storyboards.js";
export function ensureReferenceIds(clips) {
const seen = new Set();
for (const clip of clips) {
if (!clip.sectionId || seen.has(clip.sectionId)) clip.sectionId = crypto.randomUUID();
seen.add(clip.sectionId);
clip.references = structuredClone(clip.references || { mode: "defaults", asset_ids: [] });
}
}
export function loadReferences(editor) {
const raw = editor.widgets.referenceSchedule?.value;
const document = raw ? JSON.parse(raw) : { version: 1, sections: [], assets: {} };
if (document.version !== 1) throw new Error("Unsupported reference schedule version.");
if (document.sections.length && document.sections.length !== editor.clips.length) {
throw new Error("Reference sections no longer match the timeline. Restore the matching timeline before editing.");
}
editor.referenceAssets = document.assets || {};
editor.clips.forEach((clip, index) => {
const saved = document.sections[index];
if (saved) {
clip.sectionId = saved.id;
clip.references = { mode: saved.mode, asset_ids: [...saved.asset_ids] };
}
});
ensureReferenceIds(editor.clips);
}
export function serializeReferences(editor) {
ensureReferenceIds(editor.clips);
if (editor.widgets.referenceSchedule) editor.widgets.referenceSchedule.value = JSON.stringify({
version: 1,
assets: editor.referenceAssets || {},
sections: editor.clips.map(clip => ({ id: clip.sectionId, ...clip.references })),
});
}
export function mountReferences(editor) {
const client = new PromptWriterClient();
const panel = document.createElement("details");
panel.className = "flbps-reference-panel";
panel.innerHTML = `<summary>Section references</summary><div>
<select aria-label="Section reference mode"><option value="defaults">Planner defaults</option><option value="custom">Custom references</option><option value="none">No references</option></select>
<button type="button" data-ref="add">Add media</button><button type="button" data-ref="apply">Apply to selected sections</button>
<input type="file" accept="image/*,video/*,audio/*" multiple hidden>
<p>Custom references replace planner defaults. The timeline soundtrack is unchanged. Picture and Video tags each start at 1 within this section.</p>
<div data-ref="items"></div><select data-ref="library" aria-label="Add existing reference"><option value="">Add from library…</option></select>
</div>`;
editor.clipInspector.append(panel);
const mode = panel.querySelector("select");
const input = panel.querySelector("input");
const items = panel.querySelector('[data-ref="items"]');
const library = panel.querySelector('[data-ref="library"]');
const change = (label, update) => {
if (!editor.selectedClip()) return;
try {
ensureReferenceWiring(editor);
editor.runEdit(label, () => {
update(editor.selectedClip());
editor.serialize();
editor.syncInspector();
editor.scheduleDraw();
});
} catch (error) { editor.showError(error.message); }
};
mode.onchange = () => change("Change section references", clip => {
clip.references = { mode: mode.value, asset_ids: mode.value === "custom" ? (clip.references?.asset_ids || []) : [] };
});
panel.querySelector('[data-ref="apply"]').onclick = () => change("Apply section references", clip => {
for (const index of editor.selectedClipIndices()) editor.clips[index].references = structuredClone(clip.references);
});
panel.querySelector('[data-ref="add"]').onclick = () => input.click();
input.onchange = async () => {
const sectionId = editor.selectedClip()?.sectionId;
const files = [...input.files];
input.value = "";
try {
ensureReferenceWiring(editor);
const assets = [];
for (const file of files) {
const kind = file.type.split("/")[0];
if (!["image", "audio", "video"].includes(kind) || file.size > 512 * 1024 * 1024) {
throw new Error("Choose image, audio, or video media under 512 MB.");
}
const media = await client.uploadImage(file, "fl-prompt-references");
assets.push([crypto.randomUUID(), { ...media, kind, label: file.name }]);
}
editor.runEdit("Add section references", () => {
Object.assign(editor.referenceAssets, Object.fromEntries(assets));
const clip = editor.clips.find(value => value.sectionId === sectionId);
if (clip) clip.references = { mode: "custom", asset_ids: [...(clip.references?.asset_ids || []), ...assets.map(([id]) => id)] };
editor.serialize();
editor.syncInspector();
editor.scheduleDraw();
});
} catch (error) { editor.showError(error.message); }
};
library.onchange = () => {
if (!library.value) return;
change("Add library reference", clip => {
clip.references = { mode: "custom", asset_ids: [...new Set([...(clip.references?.asset_ids || []), library.value])] };
});
};
editor.syncReferences = () => {
const clip = editor.selectedClip();
panel.hidden = !clip;
if (!clip) return;
const reference = clip.references || { mode: "defaults", asset_ids: [] };
panel.querySelector("summary").textContent = `Section references · ${reference.mode === "custom" ? `${reference.asset_ids.length} custom` : reference.mode === "none" ? "None" : "Planner defaults"}`;
mode.value = reference.mode;
items.replaceChildren();
let picture = 0;
let video = 0;
const ids = [...reference.asset_ids].sort((a, b) => {
const order = { image: 0, video: 1, audio: 2 };
return order[editor.referenceAssets?.[a]?.kind] - order[editor.referenceAssets?.[b]?.kind];
});
for (const id of ids) {
const asset = editor.referenceAssets[id];
if (!asset) continue;
const row = document.createElement("div");
row.className = "flbps-reference-item";
const preview = document.createElement(asset.kind === "image" ? "img" : asset.kind);
preview.src = client.imageUrl(asset, asset.kind === "image");
if (asset.kind !== "image") preview.controls = true;
else preview.onclick = () => window.open(client.imageUrl(asset, false), "_blank", "noopener");
const label = document.createElement("span");
label.textContent = `${asset.kind === "image" ? `<Picture ${++picture}> ` : asset.kind === "video" ? `<Video ${++video}> ` : "Audio reference: "}${asset.label || asset.filename}`;
row.append(preview, label);
for (const [text, delta] of [["↑", -1], ["↓", 1], ["Remove", 0]]) {
const button = document.createElement("button");
button.textContent = text;
button.type = "button";
button.onclick = () => change("Edit reference order", target => {
const values = [...target.references.asset_ids];
const index = values.indexOf(id);
if (!delta) values.splice(index, 1);
else if (index + delta >= 0 && index + delta < values.length) [values[index], values[index + delta]] = [values[index + delta], values[index]];
target.references = { mode: "custom", asset_ids: values };
});
row.append(button);
}
items.append(row);
}
library.replaceChildren(new Option("Add from library…", ""));
for (const [id, asset] of Object.entries(editor.referenceAssets || {})) {
if (!reference.asset_ids.includes(id)) library.append(new Option(asset.label || asset.filename, id));
}
};
}
@@ -1,4 +1,5 @@
import { api } from "../../../../scripts/api.js";
import { loadReferences, serializeReferences, mountReferences } from "./audio_prompt_references.js";
import {
cropTimes,
cropTimesWithValues,
@@ -509,6 +510,7 @@ export class BeatPromptSequencer {
this.songLabelEditor = this.root.querySelector('[data-role="song-label-editor"]');
this.inspector = this.root.querySelector('[data-role="inspector"]');
this.clipInspector = this.root.querySelector('[data-role="clip-inspector"]');
mountReferences(this);
this.songInspector = this.root.querySelector('[data-role="song-inspector"]');
this.lyricsInspector = this.root.querySelector('[data-role="lyrics-inspector"]');
this.activeLaneEl = this.root.querySelector('[data-role="active-lane"]');
@@ -822,6 +824,7 @@ export class BeatPromptSequencer {
for (const name of HISTORY_WIDGETS) widgetValues[name] = cloneHistoryValue(this.widgets?.[name]?.value);
return {
clips: cloneHistoryValue(this.clips || []),
referenceAssets: cloneHistoryValue(this.referenceAssets || {}),
songMapOverrides: cloneHistoryValue(this.songMapOverrides),
lyricsTimeline: cloneHistoryValue(lyricsTimelineForStorage(this.lyricsTimeline)),
envelopeSlots: cloneHistoryValue(this.envelopeSlots || [null, null, null]),
@@ -835,6 +838,7 @@ export class BeatPromptSequencer {
if (!state) return "";
return JSON.stringify({
clips: state.clips,
referenceAssets: state.referenceAssets,
songMapOverrides: state.songMapOverrides,
lyricsTimeline: state.lyricsTimeline,
envelopeSlots: state.envelopeSlots,
@@ -983,6 +987,8 @@ export class BeatPromptSequencer {
if (this.widgets[name]) this.widgets[name].value = cloneHistoryValue(value);
}
this.clips = normalizeCrossfades(cloneHistoryValue(state.clips || []));
this.referenceAssets = cloneHistoryValue(state.referenceAssets || {});
serializeReferences(this);
this.songMapOverrides = normalizeSongMapOverrides(cloneHistoryValue(state.songMapOverrides));
this.lyricsTimeline = normalizeLyricsTimeline(cloneHistoryValue(state.lyricsTimeline));
this.envelopeSlots = cloneHistoryValue(state.envelopeSlots || [null, null, null]);
@@ -2490,6 +2496,7 @@ export class BeatPromptSequencer {
}
restoreRenderGroups() {
loadReferences(this);
try {
normalizeRenderGroups(loadRenderGroups(
this.clips,
@@ -2565,6 +2572,8 @@ export class BeatPromptSequencer {
crossfade: Math.round(finiteNumber(section.crossfade_frames)),
prompt: String(section.prompt || ""),
renderGroup: section.render_group ?? null,
sectionId: section.section_id,
references: section.references,
}));
}
@@ -3000,6 +3009,7 @@ export class BeatPromptSequencer {
}
syncInspector() {
this.syncReferences?.();
this.syncSongInspector();
this.syncLyricsInspector();
const clip = this.selectedClip();
@@ -3841,6 +3851,14 @@ export class BeatPromptSequencer {
Math.round(this.defaultFadeIn()),
Math.round(this.defaultFadeOut()),
));
const previousRanges = new Map(this.clips.map(clip => [`${clip.start}:${clip.end}`, clip]));
for (const clip of clips) {
const previous = previousRanges.get(`${clip.start}:${clip.end}`);
if (previous) {
clip.sectionId = previous.sectionId;
clip.references = structuredClone(previous.references);
}
}
this.widgets.timeUnit.value = "frames";
this.clips = clips;
this.select(clips.length ? 0 : -1);
@@ -3866,6 +3884,7 @@ export class BeatPromptSequencer {
if (this.rawInvalid || this.migrationPending || !this.widgets.timeline) return;
normalizeRenderGroups(this.clips);
this.widgets.timeline.value = serializeTimeline(this.clips);
serializeReferences(this);
if (this.widgets.renderGroups) {
this.widgets.renderGroups.value = serializeRenderGroups(this.clips);
}
@@ -4416,6 +4435,10 @@ export class BeatPromptSequencer {
groupSelectionError() {
const indices = this.selectedClipIndices();
if (indices.length < 2) return "Select at least two prompt blocks.";
const references = JSON.stringify(this.clips[indices[0]].references || { mode: "defaults", asset_ids: [] });
if (indices.some(index => JSON.stringify(this.clips[index].references || { mode: "defaults", asset_ids: [] }) !== references)) {
return "Grouped renders must share the same ordered references. Match their selections first.";
}
for (let position = 1; position < indices.length; position++) {
const previousIndex = indices[position - 1];
const index = indices[position];
@@ -4705,6 +4728,7 @@ export class BeatPromptSequencer {
const fadeOut = Math.min(clip.fadeOut, duplicateDuration - fadeIn);
this.clips.splice(this.selectedIndex + 1, 0, {
...clip,
sectionId: crypto.randomUUID(),
start,
end,
fadeIn,
@@ -4786,6 +4810,7 @@ export class BeatPromptSequencer {
const fadeOut = Math.min(source.fadeOut, duration - fadeIn);
this.clips.splice(insertionIndex, 0, {
...source,
sectionId: crypto.randomUUID(),
start,
end,
fadeIn,
@@ -4817,6 +4842,7 @@ export class BeatPromptSequencer {
};
const second = {
...clip,
sectionId: crypto.randomUUID(),
start: split,
fadeIn: 0,
fadeOut: Math.min(clip.fadeOut, clip.end - split),
@@ -5808,6 +5834,7 @@ export class BeatPromptSequencer {
const syncEnvelopePreviews = this.staticDirty;
this.draw();
if (syncEnvelopePreviews) this.syncEnvelopePreviews();
else this.updateEnvelopePlayheads();
});
}
@@ -6927,6 +6954,7 @@ export class BeatPromptSequencer {
ctx.drawImage(this.staticCanvas, 0, 0);
ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
this.drawWriterActivity(ctx);
this.renderStoryboardThumbnails?.();
this.drawGuidesAndPlayhead(ctx, cssWidth, layout);
this.positionSongLabelEditor();
}
@@ -1,6 +1,47 @@
const STYLE_ID = "fl-beat-prompt-sequencer-styles";
const STYLES = `
.flbps-reference-panel, .flbps-storyboards { padding: 8px; border-top: 1px solid #34343d; font-size: 12px; }
.flbps-storyboards { max-height: 42vh; overflow: auto; flex-shrink: 0; }
.flbps-storyboards button, .flbps-storyboards input, .flbps-storyboards select, .flbps-reference-panel button, .flbps-reference-panel select { font: inherit; }
.flbps-moodboard-slot button { max-width: 100%; overflow-wrap: anywhere; }
.flbps-storyboards summary, .flbps-reference-panel summary { cursor: pointer; padding: 4px; }
.flbps-moodboard-slots { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 5px; margin: 6px 0; }
.flbps-moodboard-slot { min-width: 0; border: 1px solid #46536b; background:#19202c; padding: 7px; border-radius: 9px; display:flex; flex-direction:column; gap:6px; }
.flbps-moodboard-slot img { width: 100%; height: 88px; object-fit: contain; cursor: zoom-in; }
.flbps-moodboard-actions { display:flex; flex-wrap:wrap; gap:4px; align-items:center; }
.flbps-moodboard-slot > button { min-height:110px; border-style:dashed; }
.flbps-moodboard-slot input:not([type=checkbox]), .flbps-storyboards textarea { width: 100%; box-sizing: border-box; }
.flbps-reference-item { display: flex; align-items: center; gap: 5px; margin: 5px 0; }
.flbps-reference-item img, .flbps-reference-item video { width: 85px; height: 60px; object-fit: contain; }
.flbps-reference-item span { flex: 1; overflow-wrap: anywhere; }
.flbps-timeline-references { position: absolute; inset: 0; overflow: hidden; pointer-events: none; }
.flbps-timeline-reference-row { position: absolute; display: flex; align-items: flex-start; gap: 6px; height: 94px; overflow: hidden; color: #c4cbd8; font-size: 11px; }
.flbps-image-card { position:relative; min-width:0; border:1px solid #4b617d; border-radius:7px; background:#111c2b; overflow:hidden; }
.flbps-timeline-reference-row .flbps-image-card { flex:0 0 104px; pointer-events:auto; }
.flbps-timeline-reference-row img { display:block; width:104px; height:62px; object-fit:contain; cursor:zoom-in; }
.flbps-image-card.loading img { opacity:0; }
.flbps-image-actions { display:flex; gap:5px; justify-content:center; padding:3px; border-top:1px solid #34455c; }
.flbps-image-loading { position:absolute; inset:0; display:flex; align-items:center; justify-content:center; flex-direction:column; gap:5px; font-size:10px; pointer-events:none; }
.loading .flbps-image-loading::before, .flbps-generation-pending::before { content:''; display:block; width:17px; height:17px; border:2px solid #435773; border-top-color:#8bceff; border-radius:50%; animation:flbps-image-spin .9s linear infinite; }
.flbps-generation-pending { flex:0 0 104px; height:88px; display:flex; flex-direction:column; align-items:center; justify-content:center; gap:7px; border:1px dashed #6691b8; border-radius:7px; background:#193049; font-size:10px; pointer-events:auto; }
@keyframes flbps-image-spin { to { transform:rotate(360deg); } }
@media(prefers-reduced-motion:reduce) { .loading .flbps-image-loading::before,.flbps-generation-pending::before { animation:none; } }
.flbps-image-dialog { width:min(1200px,90vw); max-width:94vw; height:min(820px,88vh); max-height:92vh; border:1px solid #526b88; border-radius:14px; padding:0; background:#101722; color:#e3edfa; box-shadow:0 24px 100px #000a; }
.flbps-image-dialog[open] { display:grid; grid-template-columns:minmax(0,1fr) minmax(260px,32%); }
.flbps-image-dialog::backdrop { background:#050a13c9; }
.flbps-image-view { position:relative; display:flex; align-items:center; justify-content:center; min-width:0; overflow:hidden; background:#080d15; padding:14px; }
.flbps-image-view img { max-width:100%; max-height:100%; object-fit:contain; }
.flbps-image-dialog aside { padding:20px; overflow:auto; border-left:1px solid #34445a; font:12px/1.5 system-ui; }
.flbps-image-dialog h2 { font-size:18px; margin:0 0 12px; }
.flbps-image-dialog h4 { color:#85b6e5; margin:18px 0 5px; }
.flbps-image-dialog p { white-space:pre-wrap; overflow-wrap:anywhere; margin:0; }
.flbps-image-close { background:#25364c; color:white; border:1px solid #567797; padding:7px 12px; border-radius:6px; cursor:pointer; }
@media(max-width:650px) { .flbps-image-dialog[open] { grid-template-columns:1fr; grid-template-rows:55% 45%; } }
.flbps-timeline-reference-row button { flex: 0 0 auto; width: 23px; height: 25px; padding: 0; border: 1px solid #64748b; border-radius: 4px; background: #202735; color: #e2e8f0; cursor: pointer; pointer-events: auto; }
.flbps-timeline-reference-row button:disabled { opacity: .45; cursor: wait; }
.flbps-storyboards button, .flbps-reference-panel button, .flbps-storyboards select, .flbps-reference-panel select { background: #282831; color: #ededf3; border: 1px solid #484853; border-radius: 4px; padding: 4px; margin: 2px; }
.flbps-storyboards textarea, .flbps-storyboards input:not([type=checkbox]) { background: #181820; color: #ededf3; border: 1px solid #484853; }
.flbps-root {
height: 100%;
min-height: 0;
@@ -946,14 +987,6 @@ const STYLES = `
scrollbar-color: #35353d transparent;
}
.flbps-writer-thread { display: flex; flex-direction: column; gap: 18px; }
.flbps-writer-welcome { max-width: 330px; margin: auto; padding: 22px 4px 14px; text-align: center; animation: flbps-writer-rise .24s ease-out; }
.flbps-writer-welcome[hidden] { display: none; }
.flbps-writer-welcome-mark { width: 48px; height: 48px; display: grid; place-items: center; margin: 0 auto 13px; color: #fff; background: linear-gradient(145deg, var(--writer-accent), #334155); border: 1px solid rgba(255, 255, 255, .18); border-radius: 15px; box-shadow: 0 15px 35px var(--writer-accent-soft), inset 0 1px rgba(255, 255, 255, .25); font-size: 16px; font-weight: 800; }
.flbps-writer-welcome h3 { margin: 0 0 7px; color: #f4f4f5; font-size: 13px; letter-spacing: -.02em; }
.flbps-writer-welcome p { max-width: 285px; margin: 0 auto 16px; color: #85858f; font-size: 8.5px; line-height: 1.55; }
.flbps-writer-starters { display: grid; grid-template-columns: 1fr 1fr; gap: 7px; }
.flbps-writer-starters button { min-height: 38px; padding: 8px; color: #b9b9c2; background: rgba(34, 34, 40, .75); border: 1px solid var(--writer-border); border-radius: 9px; font-size: 8px; line-height: 1.25; cursor: pointer; }
.flbps-writer-starters button:hover { color: #fff; background: var(--writer-accent-soft); border-color: color-mix(in srgb, var(--writer-accent), transparent 48%); transform: translateY(-1px); }
.flbps-writer-message { max-width: none; padding: 0; background: transparent !important; border: 0 !important; border-radius: 0; animation: flbps-writer-rise .18s ease-out; }
.flbps-writer-message.user { width: min(88%, 315px); align-self: flex-end; }
.flbps-writer-message.assistant { width: 100%; align-self: stretch; }
@@ -1078,7 +1111,7 @@ const STYLES = `
}
.flbps-writer-settings-card textarea { min-height: 105px; resize: vertical; line-height: 1.5; }
.flbps-writer-inline { display: flex; gap: 5px; }
.flbps-writer-inline input { min-width: 0; flex: 1; }
.flbps-writer-inline input, .flbps-writer-inline select { min-width: 0; flex: 1; }
.flbps-writer-inline button, .flbps-writer-subscription button { padding: 0 8px; color: #b7b7c0; background: #29292f; border: 1px solid var(--writer-border); border-radius: 6px; font-size: 7px; cursor: pointer; }
.flbps-writer-setting-row { display: grid; grid-template-columns: 1fr 1fr; gap: 7px; }
.flbps-writer-subscription { padding: 8px; background: rgba(255, 255, 255, .03); border: 1px solid var(--writer-border); border-radius: 7px; }
+441
View File
@@ -0,0 +1,441 @@
import { api, ComfyApi } from "../../../../scripts/api.js";
import { app } from "../../../../scripts/app.js";
const PREFIX = "/fl/audio-prompt-timeline/storyboards";
export function storyboardThumbnailUrl(image) {
return api.apiURL(`${PREFIX}/thumbnail?${new URLSearchParams({filename:image.filename,subfolder:image.subfolder||"",type:image.type||"input"})}`);
}
export function storyboardContinuity(actions) {
const brief = "These sections belong to ONE continuous production. Keep recurring characters, costumes, visual style and environments coherent. Do not invent a new design for each section.\n"
+ actions.map((action, index) => `Section ${index+1}: ${action.prompt}`).join("\n\n");
if (brief.length > 32000) throw new Error("The shared storyboard brief is too long. Generate a smaller group of sections.");
return brief;
}
export async function prepareStoryboardQueue() {
const stores = app.extensionManager._p._s;
const auth = stores.get("auth");
const keyAuth = stores.get("apiKeyAuth");
if (!auth || !keyAuth) throw new Error("ComfyUI Partner authentication is not ready. Refresh ComfyUI and sign in.");
const authToken = await auth.getAuthToken();
const apiKey = keyAuth.getApiKey();
if (!authToken && !apiKey) throw new Error("Sign in to ComfyUI before generating a Partner Node image.");
// Match ComfyApp's queue credentials without modifying the shared API or active graph.
const submission = { clientId: api.clientId, authToken, apiKey, fetchApi: api.fetchApi.bind(api) };
return (graph, jobId) => ComfyApi.prototype.queuePrompt.call(submission, 0, {
output: graph, workflow: { nodes: [], links: [], extra: { storyboard_id: jobId } },
});
}
async function request(path = "", body) {
const response = await api.fetchApi(PREFIX + path, body === undefined ? {} : {
method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body),
});
const result = await response.json();
if (!response.ok) throw new Error(result.error || "Storyboard request failed.");
return result;
}
export function ensureReferenceWiring(editor) {
const node = editor.node;
const graph = node.graph;
if (!graph) throw new Error("The scheduler must belong to a workflow before using references.");
const planners = graph._nodes.filter(target => target.type === "FL_MiniMaxH3BeatShotPlanner"
&& target.inputs.some(input => input.name === "prompt_schedule" && graph.links[input.link]?.origin_id === node.id));
if (!planners.length) throw new Error("Connect this scheduler directly to an H3 Beat Shot Planner before generating references.");
let library = graph._nodes.find(target => target.type === "FL_Prompt_Reference_Library"
&& target.inputs.some(input => input.name === "prompt_schedule" && graph.links[input.link]?.origin_id === node.id));
if (!library) {
library = LiteGraph.createNode("FL_Prompt_Reference_Library");
if (!library) throw new Error("Restart ComfyUI to load FL Prompt Reference Library.");
library.pos = [node.pos[0] + node.size[0] + 40, node.pos[1] + 120];
graph.add(library);
node.connect(0, library, 0);
}
for (const planner of planners) {
const slot = planner.inputs.findIndex(input => input.name === "reference_library");
if (slot < 0) throw new Error("Refresh ComfyUI to update the planner's reference input.");
const input = planner.inputs[slot];
if (graph.links[input.link]?.origin_id !== library.id) library.connect(0, planner, slot);
const mode = planner.widgets.find(widget => widget.name === "visual_reference_mode");
if (mode && mode.value !== "full") { mode.value = "full"; mode.callback?.(mode.value); }
}
graph.change();
}
export function timelineReferenceImages(clip, assets) {
if (clip.references?.mode !== "custom") return [];
const groups = new Map();
for (const id of clip.references.asset_ids) {
const asset = assets[id];
if (asset?.kind !== "image") continue;
const key = asset.storyboard_id || id;
if (!groups.has(key)) groups.set(key, { key, image: asset.source || asset, assetIds: [] });
groups.get(key).assetIds.push(id);
}
return [...groups.values()];
}
export function mountStoryboards(writer) {
const editor = writer.editor;
const boards = writer.nodeSettings.moodboards;
const root = document.createElement("section");
root.className = "flbps-storyboards";
root.innerHTML = `<div class="flbps-moodboard-slots"></div>
<small>Generate sections concurrently with shared character and style references. All requested images may spend ComfyUI credits at once; provider limits still apply. Checked moodboards are sent to Google. Rerolls retain their original references.</small>
<div data-story="status" role="status"></div>`;
writer.root.querySelector(".flbps-writer-topbar").after(root);
const slots = root.querySelector(".flbps-moodboard-slots");
const status = root.querySelector('[data-story="status"]');
const error = task => Promise.resolve().then(task).catch(value => { status.textContent = value.message; writer.showError(value.message); });
const busy = new Set();
const attachmentErrors = new Map();
let jobs = [];
let disposed = false;
let timer;
let checking = false;
let renderKey = "";
const handled = new Set(writer.nodeSettings.storyboardResults || []);
const overlay = document.createElement("div");
overlay.className = "flbps-timeline-references";
editor.canvas.parentElement.append(overlay);
overlay.onwheel = event => editor.onWheel(event);
let viewer = null;
function openPreview(image, metadata = {}) {
viewer?.close();
const focus = document.activeElement;
const dialog = document.createElement("dialog");viewer=dialog;
dialog.className="flbps-image-dialog";dialog.setAttribute("aria-label","Storyboard image and metadata");
const visual=document.createElement("div"),details=document.createElement("aside"),full=document.createElement("img"),close=document.createElement("button");
visual.className="flbps-image-view loading";full.alt=metadata.title||"Reference image";
const loading=document.createElement("span");loading.className="flbps-image-loading";loading.textContent="Loading original…";
full.onload=()=>{visual.className="flbps-image-view";loading.remove();field("Dimensions",`${full.naturalWidth} × ${full.naturalHeight}`);};
full.onerror=()=>{visual.className="flbps-image-view";loading.textContent="Original image unavailable.";};
close.textContent="Close ×";close.className="flbps-image-close";close.onclick=()=>dialog.close();
function field(label,value){if(value===undefined||value===null||value==="")return;const h=document.createElement("h4"),p=document.createElement("p");h.textContent=label;p.textContent=String(value);details.append(h,p);}
const title=document.createElement("h2");title.textContent=metadata.title||"Reference image";details.append(title,close);
field("File",image.filename);field("Location",`${image.type||"input"}/${image.subfolder||""}`);
field("Section",metadata.section);field("State",metadata.job?.state);field("Model",metadata.job?"Nano Banana 2 (Gemini 3.1 Flash Image)":null);
const spec=metadata.job?.spec;field("Grid",spec?`${spec.grid} × ${spec.grid}`:null);field("Requested size",spec?`${spec.resolution} · ${spec.aspect_ratio}`:null);
field("Prompt",spec?.prompt);field("Continuity brief",spec?.continuity);field("Reference roles",spec?.moodboards?.map((b,i)=>`${i+1}. ${b.role||b.filename}`).join("\n"));field("Role",metadata.role);
visual.append(loading,full);dialog.append(visual,details);document.body.append(dialog);
dialog.onclick=e=>{if(e.target===dialog)dialog.close();};dialog.onclose=()=>{full.removeAttribute("src");dialog.remove();if(viewer===dialog)viewer=null;if(focus?.isConnected)focus.focus();};
dialog.onkeydown=e=>e.stopPropagation();
full.src=writer.client.imageUrl(image,false);dialog.showModal();close.focus();
}
function thumbnail(image, label, click) {
const card=document.createElement("div");card.className="flbps-image-card loading";
const img=document.createElement("img");img.alt=label;img.loading="lazy";img.decoding="async";img.draggable=false;img.tabIndex=0;img.setAttribute("role","button");
const state=document.createElement("span");state.className="flbps-image-loading";state.textContent="Loading…";
img.onload=()=>{card.className="flbps-image-card";state.remove();};img.onerror=()=>{card.className="flbps-image-card failed";state.textContent="Preview unavailable";};
img.onclick=click;img.onkeydown=e=>{if(e.key==="Enter"||e.key===" "){e.preventDefault();e.stopPropagation();click();}};
img.src=storyboardThumbnailUrl(image);card.append(img,state);return card;
}
function renderSlots() {
slots.replaceChildren();
for (let index = 0; index < 4; index++) {
const board = boards[index];
const slot = document.createElement("div");
slot.className = "flbps-moodboard-slot";
const choose = document.createElement("button");
choose.type = "button";
choose.textContent = board ? "Replace" : `+ Moodboard ${index+1}`;
const input = document.createElement("input");
input.type = "file";
input.accept = "image/png,image/jpeg,image/webp,image/gif";
input.hidden = true;
const upload = async file => {
if (!file || !["image/png", "image/jpeg", "image/webp", "image/gif"].includes(file.type) || file.size > 32*1024*1024) {
throw new Error("Choose a PNG, JPEG, WebP or GIF under 32 MB.");
}
const image = await writer.client.uploadImage(file, `fl-beat-writer/${writer.nodeSettings.schedulerId}`);
boards[index] = { ...image, role: board?.role || "", selected: true, kind: "image", label: file.name, originalName: file.name, mimeType: file.type };
writer.saveNodeSettings();
renderSlots();
};
choose.onclick = () => input.click();
input.onchange = () => error(() => upload(input.files[0]));
slot.ondragover = event => event.preventDefault();
slot.ondrop = event => { event.preventDefault(); event.stopPropagation(); error(() => upload(event.dataTransfer.files[0])); };
slot.tabIndex = 0;
slot.onpaste = event => {
const file = [...event.clipboardData.files].find(value => value.type.startsWith("image/"));
if (file) { event.preventDefault(); event.stopPropagation(); error(() => upload(file)); }
};
if (board) {
const image = thumbnail(board,board.label || `Moodboard ${index+1}`,()=>openPreview(board,{title:`Moodboard ${index+1}`,role:board.role}));
const role = document.createElement("input");
role.placeholder = "Role: style, character…";
role.title = "How this image should guide generation: character identity, costume, palette, or style.";
role.value = board.role;
role.maxLength = 500;
role.onchange = () => { board.role = role.value; writer.saveNodeSettings(); };
const selected = document.createElement("input");
selected.type = "checkbox";
selected.checked = board.selected;
selected.title = "Include in the next storyboard generation";
selected.onchange = () => { board.selected = selected.checked; writer.saveNodeSettings(); };
const clear = document.createElement("button");
clear.textContent = "Clear";
clear.type = "button";
clear.onclick = () => { boards[index] = null; writer.saveNodeSettings(); renderSlots(); };
const use = document.createElement("label");
use.append(selected, document.createTextNode("Use"));
const actions=document.createElement("div");actions.className="flbps-moodboard-actions";actions.append(use,clear,choose);
slot.append(image, role, actions);
}
if(!board)slot.append(choose);
slot.append(input);
slots.append(slot);
}
}
function markHandled(job) {
handled.add(job.id);
writer.nodeSettings.storyboardResults = [...handled];
writer.saveNodeSettings();
}
async function install(job, explicit = false) {
if (handled.has(job.id)) return;
const clip = editor.clips.find(value => value.sectionId === job.spec.section_id);
if (!clip) return;
if (clip.references?.asset_ids.some(id => editor.referenceAssets[id]?.storyboard_id === job.id)) {
markHandled(job);
return;
}
const revision = await promptRevision(clip);
if (!explicit && revision !== job.spec.revision) {
attachmentErrors.set(job.id, "Section changed. Saved image is ready to attach; current references were kept.");
return;
}
ensureReferenceWiring(editor);
if (!job.result.assets) job = await request(`/${job.id}/extract`, {});
if (disposed) return;
// Extraction is asynchronous: recheck before applying to a section the user may have edited.
if (!editor.clips.includes(clip) || await promptRevision(clip) !== revision) {
attachmentErrors.set(job.id, "Section changed during attachment. Review and attach the saved image again.");
return;
}
editor.runEdit("Use generated storyboard references", () => {
Object.assign(editor.referenceAssets, job.result.assets);
clip.references = { mode: "custom", asset_ids: Object.keys(job.result.assets) };
editor.serialize();
editor.syncInspector();
});
markHandled(job);
attachmentErrors.delete(job.id);
status.textContent = "Storyboard attached. Its panels are now this section's video references.";
editor.scheduleDraw();
}
async function refresh() {
if (disposed || checking) return;
checking = true;
try {
const listed = await request(`?scheduler_id=${encodeURIComponent(writer.nodeSettings.schedulerId)}`);
jobs = [...listed, ...jobs.filter(job => job.state === "submitted" && !listed.some(value => value.id === job.id))];
for (let index = 0; index < jobs.length && !disposed; index++) {
let job = jobs[index];
if (["submitted", "unknown"].includes(job.state)) job = await request(`/${job.id}/refresh`, {});
jobs[index] = job;
if (job.state === "complete") {
try { await install(job); }
catch (failure) { attachmentErrors.set(job.id, failure.message); }
}
}
if (["failed", "unknown"].includes(jobs[0]?.state)) status.textContent = jobs[0].result.error;
editor.scheduleDraw();
} catch (failure) {
status.textContent = failure.message;
} finally {
checking = false;
if (!disposed) timer = setTimeout(refresh, 3000);
}
}
async function generate(clip, prompt, grid, requestKey, context = {}) {
if (busy.has(clip.sectionId) || jobs.some(job => job.spec.section_id === clip.sectionId && job.state === "submitted")) {
throw new Error("This section already has an active image request. Wait for it to finish before generating again.");
}
busy.add(clip.sectionId);
editor.scheduleDraw();
try {
ensureReferenceWiring(editor);
const revision = await promptRevision(clip);
const submit = await prepareStoryboardQueue();
const job = await request("", {
scheduler_id: writer.nodeSettings.schedulerId, section_id: clip.sectionId,
revision, prompt, grid, resolution: "2K", aspect_ratio: "16:9",
moodboards: context.moodboards || boards.filter(board => board?.selected), request_key: requestKey,
continuity: context.continuity || "",
});
if (job.state !== "proposed") return;
const { graph } = await request(`/${job.id}/submit`, {});
try {
const receipt = await submit(graph, job.id);
await request(`/${job.id}/receipt`, { prompt_id: receipt.prompt_id });
status.textContent = "Storyboard queued in ComfyUI. It will attach to its section when finished.";
jobs.unshift({ ...job, state: "submitted" });
} catch (failure) {
throw new Error(`Submission outcome may be unknown. Check ComfyUI history before rerolling. ${failure.message}`);
}
} finally {
busy.delete(clip.sectionId);
editor.scheduleDraw();
}
}
async function generateBatch(items, context) {
if (items.length === 1) {
const {clip,action,key} = items[0];
return generate(clip,action.prompt,action.grid,key,context);
}
if (items.some(({clip}) => busy.has(clip.sectionId) || jobs.some(j => j.spec.section_id === clip.sectionId && j.state === "submitted"))) {
throw new Error("A section already has an active storyboard request. Wait before generating it again.");
}
items.forEach(({clip}) => busy.add(clip.sectionId));editor.scheduleDraw();
try {
ensureReferenceWiring(editor);
const revisions = await Promise.all(items.map(({clip}) => promptRevision(clip)));
const submit = await prepareStoryboardQueue();
const created = await Promise.all(items.map(async ({clip,action,key},index) => request("", {
scheduler_id: writer.nodeSettings.schedulerId, section_id: clip.sectionId,
revision: revisions[index], prompt: action.prompt, grid: action.grid,
resolution: "2K", aspect_ratio: "16:9", request_key: key, ...context,
})));
const proposed = created.filter(job => job.state === "proposed");
if (!proposed.length) return;
const {graph} = await request("/batch/submit", {job_ids: proposed.map(job => job.id)});
try {
const receipt = await submit(graph, proposed.map(job => job.id).join(","));
jobs.unshift(...proposed.map(job => ({...job,state:"submitted"})));
await Promise.all(proposed.map(job => request(`/${job.id}/receipt`, {prompt_id: receipt.prompt_id})));
status.textContent = `${proposed.length} storyboards queued together for asynchronous generation. Shared references and continuity brief applied.`;
} catch (failure) {
throw new Error(`Batch submission outcome may be unknown. Check ComfyUI history before rerolling; requests may be billed. ${failure.message}`);
}
} finally {items.forEach(({clip}) => busy.delete(clip.sectionId));editor.scheduleDraw();}
}
editor.renderStoryboardThumbnails = () => {
const rows = (editor.clipRects || []).map(rect => {
const clip = editor.clips[rect.index];
return { rect, clip, images: timelineReferenceImages(clip, editor.referenceAssets || {}),
ready: jobs.filter(job => job.spec.section_id === clip.sectionId && job.state === "complete" && job.result.source && !handled.has(job.id)),
pending: busy.has(clip.sectionId) || jobs.some(job => job.spec.section_id === clip.sectionId && job.state === "submitted") };
});
const key = JSON.stringify(rows.map(row => [row.clip.sectionId, row.images, row.pending, row.ready.map(job => [job.id, attachmentErrors.get(job.id)]), row.images.map(item => jobs.some(job => job.id === item.key))]));
if (key !== renderKey) {
renderKey = key;
overlay.replaceChildren();
for (const { clip, images, pending, ready } of rows) {
const row = document.createElement("div");
row.className = "flbps-timeline-reference-row";
row.dataset.sectionId = clip.sectionId;
row.onpointerdown = event => event.stopPropagation();
row.ondblclick = event => event.stopPropagation();
for (const item of images) {
const source = jobs.find(job => job.id === item.key);
const card=thumbnail(item.image,"Active video reference",()=>openPreview(item.image,{title:"Storyboard reference",section:clip.sectionId,job:source}));
const actions=document.createElement("div");actions.className="flbps-image-actions";card.append(actions);row.append(card);
if (source) {
const reroll = document.createElement("button");
reroll.type = "button";
reroll.textContent = "↻";
reroll.title = "Reroll this storyboard (uses ComfyUI credits)";
reroll.disabled = pending;
reroll.onclick = () => error(() => generate(clip, source.spec.prompt, source.spec.grid, crypto.randomUUID(), {
continuity: source.spec.continuity || "", moodboards: source.spec.moodboards,
}));
reroll.setAttribute("aria-label","Reroll storyboard");actions.append(reroll);
}
const remove = document.createElement("button");
remove.type = "button";
remove.textContent = "×";
remove.title = "Remove this reference from the section";
remove.onclick = () => editor.runEdit("Remove timeline reference", () => {
clip.references = { mode: "custom", asset_ids: clip.references.asset_ids.filter(id => !item.assetIds.includes(id)) };
editor.serialize();
editor.syncInspector();
editor.scheduleDraw();
});
remove.setAttribute("aria-label","Remove section reference");actions.append(remove);
}
for (const job of ready) {
const card = thumbnail(job.result.source, "Saved storyboard — not attached", () => openPreview(job.result.source, {title:"Saved storyboard — not attached", section:clip.sectionId, job}));
const reason = document.createElement("small");
reason.textContent = attachmentErrors.get(job.id) || "Saved image — not attached";
reason.title = reason.textContent;
const actions = document.createElement("div");actions.className = "flbps-image-actions";
const attach = document.createElement("button");attach.textContent = "Attach";
attach.title = "Use this saved storyboard as this section's references (no generation credits)";
attach.onclick = () => error(async () => {
attach.disabled = true;
try { await install(job, true); }
finally { attach.disabled = false; editor.scheduleDraw(); }
});
const dismiss = document.createElement("button");dismiss.textContent = "×";
dismiss.title = "Dismiss this saved result; keep current references";
dismiss.onclick = () => { markHandled(job); attachmentErrors.delete(job.id); editor.scheduleDraw(); };
actions.append(attach, dismiss);card.append(reason, actions);row.append(card);
}
if (pending) {const pendingCard=document.createElement("div");pendingCard.className="flbps-generation-pending";pendingCard.setAttribute("role","status");pendingCard.textContent="Queued / generating…";row.append(pendingCard);}
overlay.append(row);
}
}
for (const { rect, clip } of rows) {
const row = [...overlay.children].find(value => value.dataset.sectionId === clip.sectionId);
if (!row) continue;
row.style.left = `${Math.max(0, rect.x + 8)}px`;
row.style.top = `${rect.y + Math.max(32, Math.min(48, rect.height - 100))}px`;
row.style.width = `${Math.max(0, rect.width - 16)}px`;
row.style.maxHeight = `${Math.max(0,rect.height-40)}px`;
}
};
function actionClip(proposal, document, updates) {
const clip = editor.clips[proposal.index];
const expected = updates?.find(value => value.index === proposal.index) || document?.boxes?.find(value => value.index === proposal.index);
if (!document?.allowed_indices?.includes(proposal.index) || !expected || clip?.sectionId !== document.sectionIds?.[proposal.index]
|| clip.start !== expected.start_frame || clip.end !== expected.end_frame || clip.prompt !== expected.prompt
|| JSON.stringify(clip.references) !== JSON.stringify(document.referenceSelections?.[proposal.index])) {
throw new Error("The section changed while the Writer was working. Ask for a fresh storyboard.");
}
return clip;
}
writer.generateStoryboards = (actions, updates, messageId) => error(async () => {
const document = writer.currentDocument;
const context = {continuity: storyboardContinuity(actions), moodboards: structuredClone(boards.filter(board => board?.selected))};
// Validate every section before creating any paid submission.
const sections = actions.map(action => ({action,clip:actionClip(action,document,updates)}));
if (new Set(sections.map(({clip})=>clip.sectionId)).size !== sections.length) throw new Error("Request each storyboard section only once per batch.");
if (disposed || !sections.length) return;
const items=sections.map(({action})=>{
const clip=actionClip(action,document,updates);return {action,clip,key:`${messageId}:${clip.sectionId}`};
});
await generateBatch(items,context);
});
writer.applyReferenceAssignments = (actions, updates) => error(async () => {
for (const action of actions) {
const clip = actionClip(action, writer.currentDocument, updates);
ensureReferenceWiring(editor);
editor.runEdit("Apply Writer references", () => {
clip.references = { mode: action.mode, asset_ids: [...action.asset_ids] };
editor.serialize();
editor.syncInspector();
});
editor.scheduleDraw();
}
});
writer.disposeStoryboards = () => {
disposed = true;
clearTimeout(timer);
viewer?.close();
delete editor.renderStoryboardThumbnails;
overlay.remove();
};
renderSlots();
refresh();
}
async function promptRevision(clip) {
const bytes = new TextEncoder().encode(JSON.stringify([clip.start, clip.end, clip.prompt, clip.references]));
return [...new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))].map(value => value.toString(16).padStart(2, "0")).join("");
}
+79 -38
View File
@@ -1,5 +1,6 @@
import { PromptWriterClient } from "./audio_prompt_writer_client.js";
import { renderWriterMarkdown } from "./audio_prompt_writer_markdown.js";
import { mountStoryboards } from "./audio_prompt_storyboards.js";
const NODE_DEFAULTS = {
guideMode: "video_prompt_guide",
@@ -7,13 +8,6 @@ const NODE_DEFAULTS = {
context: "",
};
const STARTERS = [
["Rewrite with the guide", "Rewrite every prompt box using the complete packaged prompt-writing guide while preserving the story and timing."],
["Strengthen continuity", "Strengthen visual and narrative continuity across these prompt boxes. Keep each beat distinct and actionable."],
["Make action explicit", "Make the physical action, camera behavior, and scene construction more explicit in every prompt that needs it."],
["Review first", "Review the current prompt sequence for continuity, clarity, and guide compliance. Do not edit anything yet."],
];
const MAX_CHAT_ATTACHMENTS = 8;
const MAX_CHAT_ATTACHMENT_BYTES = 32 * 1024 * 1024;
const CHAT_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
@@ -33,6 +27,8 @@ function nodeSettings(node) {
schedulerId: saved.schedulerId,
} : {}),
schedulerId: saved?.schedulerId || createId(),
moodboards: Array.from({ length: 4 }, (_, index) => saved?.moodboards?.[index] || null),
storyboardResults: Array.isArray(saved?.storyboardResults) ? saved.storyboardResults : [],
};
}
@@ -74,6 +70,8 @@ export class BeatPromptWriter {
this.nodeSettings = nodeSettings(node);
this.settings = null;
this.status = null;
this.models = [];
this.modelRequest = 0;
this.conversations = [];
this.archivedConversations = [];
this.conversationId = null;
@@ -106,6 +104,7 @@ export class BeatPromptWriter {
};
this.destroyed = false;
this.build();
mountStoryboards(this);
this.saveNodeSettings();
this.initialize();
}
@@ -140,12 +139,6 @@ export class BeatPromptWriter {
<section class="flbps-writer-view active" data-writer-view="chat">
<div class="flbps-writer-banner" data-writer-role="status" aria-live="polite"><i></i><span>Connecting...</span></div>
<div class="flbps-writer-messages" data-writer-role="messages">
<section class="flbps-writer-welcome" data-writer-role="welcome">
<div class="flbps-writer-welcome-mark">W</div>
<h3>Write the whole sequence together.</h3>
<p>Chat about the story, review the timeline, or ask Beat Writer to revise prompt boxes with the complete guide.</p>
<div class="flbps-writer-starters" data-writer-role="starters"></div>
</section>
<div class="flbps-writer-thread" data-writer-role="thread"></div>
</div>
<button class="flbps-writer-jump" data-writer-action="jump-latest" hidden>Jump to latest <span>down</span></button>
@@ -183,8 +176,9 @@ export class BeatPromptWriter {
<details class="flbps-writer-settings-card" open><summary><span>Connection</span><em data-writer-role="connection-pill">Checking</em></summary><div>
<label>Provider<select data-writer-setting="provider"></select></label>
<label data-writer-role="base-url-row">Base URL<input data-writer-setting="base-url" type="url" spellcheck="false"></label>
<label>Model<div class="flbps-writer-inline"><input data-writer-setting="model" type="text" list="flbps-writer-models" spellcheck="false"><button data-writer-action="models">Refresh</button></div></label>
<label>Model<div class="flbps-writer-inline"><select data-writer-setting="model-select" aria-label="Subscription model" hidden></select><input data-writer-setting="model" type="text" list="flbps-writer-models" spellcheck="false"><button data-writer-action="models">Refresh</button></div></label>
<datalist id="flbps-writer-models"></datalist>
<p class="flbps-writer-scope-note" data-writer-role="model-status" aria-live="polite"></p>
<label>Default reasoning<select data-writer-setting="reasoning"></select></label>
<div class="flbps-writer-setting-row"><label>Temperature<input data-writer-setting="temperature" type="number" min="0" max="2" step="0.1"></label><label>Max tokens<input data-writer-setting="max-tokens" type="number" min="256" max="32768" step="256"></label></div>
<label data-writer-role="credential-row">API key<div class="flbps-writer-inline"><input data-writer-setting="credential" type="password" autocomplete="off" placeholder="Stored in your OS keychain"><button data-writer-action="clear-credential">Clear</button></div></label>
@@ -194,7 +188,7 @@ export class BeatPromptWriter {
<details class="flbps-writer-settings-card" open><summary><span>Writing</span><em>Prompt-only</em></summary><div>
<label>Prompt guide<select data-writer-node-setting="guide-mode"><option value="video_prompt_guide">Complete packaged guide</option><option value="preserve">Preserve current format</option><option value="freeform">Freeform with guide reference</option></select></label>
<label>Story bible / persistent context<textarea data-writer-node-setting="context" placeholder="Characters, style rules, continuity, and story intent"></textarea></label>
<p class="flbps-writer-scope-note">Beat Writer can inspect attached reference images and replace prompt text in the selected scope. Images stay read-only; it cannot change timing, nodes, files, or workflow structure.</p>
<p class="flbps-writer-scope-note">Beat Writer can inspect attached images and checked moodboards, replace scoped prompts, and generate storyboard references. Generation uses ComfyUI credits and attaches panels automatically. Reference Library wiring is added when needed; timing stays unchanged.</p>
</div></details>
</div>
</section>
@@ -221,7 +215,6 @@ export class BeatPromptWriter {
this.container.appendChild(this.root);
this.messagesElement = this.root.querySelector('[data-writer-role="messages"]');
this.threadElement = this.root.querySelector('[data-writer-role="thread"]');
this.welcomeElement = this.root.querySelector('[data-writer-role="welcome"]');
this.statusElement = this.root.querySelector('[data-writer-role="status"]');
this.statusText = this.statusElement.querySelector("span");
this.composer = this.root.querySelector('[data-writer-role="composer"]');
@@ -238,6 +231,8 @@ export class BeatPromptWriter {
this.providerSelect = this.root.querySelector('[data-writer-setting="provider"]');
this.baseUrlInput = this.root.querySelector('[data-writer-setting="base-url"]');
this.modelInput = this.root.querySelector('[data-writer-setting="model"]');
this.modelSelect = this.root.querySelector('[data-writer-setting="model-select"]');
this.modelStatus = this.root.querySelector('[data-writer-role="model-status"]');
this.modelOptions = this.root.querySelector("#flbps-writer-models");
this.reasoningSelect = this.root.querySelector('[data-writer-setting="reasoning"]');
this.temperatureInput = this.root.querySelector('[data-writer-setting="temperature"]');
@@ -253,20 +248,15 @@ export class BeatPromptWriter {
this.nodeControls.scope.value = this.nodeSettings.scope;
this.nodeControls.guideMode.value = this.nodeSettings.guideMode;
this.nodeControls.context.value = this.nodeSettings.context;
const starters = this.root.querySelector('[data-writer-role="starters"]');
for (const [label, prompt] of STARTERS) {
const button = document.createElement("button");
button.type = "button";
button.dataset.writerAction = "starter";
button.dataset.prompt = prompt;
button.textContent = label;
starters.appendChild(button);
}
this.root.addEventListener("click", (event) => {
this.handleAction(event).catch((error) => this.showError(error.message));
});
this.providerSelect.addEventListener("change", () => this.applyProviderPreset());
this.modelSelect.addEventListener("change", () => {
this.modelInput.value = this.modelSelect.value;
this.populateReasoning(this.reasoningSelect.value);
});
for (const [name, control] of Object.entries(this.nodeControls)) {
control.addEventListener(name === "context" ? "input" : "change", () => {
this.nodeSettings[name] = control.value;
@@ -324,6 +314,7 @@ export class BeatPromptWriter {
if (this.destroyed) return;
this.populateSettings();
this.updateProviderBadge();
this.discoverModels(false);
await this.refreshConversations();
const resumed = await this.resumeActiveRun();
if (!resumed && this.errorElement.hidden) {
@@ -508,10 +499,6 @@ export class BeatPromptWriter {
else if (action === "toggle-menu") {
const menu = this.root.querySelector('[data-writer-role="menu"]');
menu.hidden = !menu.hidden;
} else if (action === "starter") {
this.composer.value = button.dataset.prompt || "";
this.updateComposer();
this.composer.focus();
} else if (action === "send") await this.send();
else if (action === "attach-images") this.attachmentInput.click();
else if (action === "remove-attachment") this.removePendingAttachment(Number(button.dataset.attachmentIndex));
@@ -567,7 +554,8 @@ export class BeatPromptWriter {
populateReasoning(selected) {
const preset = this.settings?.presets?.[this.providerSelect.value] || {};
const efforts = ["default", ...(preset.reasoning_efforts || [])];
const model = this.models.find((item) => item.id === this.modelInput.value);
const efforts = ["default", ...(model?.reasoningEfforts || preset.reasoning_efforts || [])];
for (const select of [this.reasoningSelect, this.reasoningComposer]) {
select.replaceChildren(...efforts.map((effort) => option(effort, effort === "default" ? "Default" : effort[0].toUpperCase() + effort.slice(1))));
select.value = efforts.includes(selected) ? selected : "default";
@@ -575,16 +563,22 @@ export class BeatPromptWriter {
}
applyProviderPreset() {
this.modelRequest += 1;
this.models = [];
const preset = this.settings.presets[this.providerSelect.value];
this.baseUrlInput.value = preset.base_url || "";
this.modelInput.value = preset.default_model || "";
this.populateReasoning("default");
this.updateProviderControls();
this.discoverModels(false);
}
updateProviderControls() {
const preset = this.settings.presets[this.providerSelect.value];
const subscription = ["codex_cli", "claude_cli"].includes(preset.type);
this.modelInput.hidden = subscription;
this.modelSelect.hidden = !subscription;
this.renderModelOptions();
this.root.querySelector('[data-writer-role="base-url-row"]').hidden = preset.type !== "openai_compatible";
this.root.querySelector('[data-writer-role="credential-row"]').hidden = subscription || (!preset.requires_key && this.providerSelect.value !== "custom");
this.root.querySelector('[data-writer-role="subscription-row"]').hidden = !subscription;
@@ -620,11 +614,41 @@ export class BeatPromptWriter {
this.toast("Stored credential cleared");
}
renderModelOptions() {
const selected = this.modelInput.value;
const choices = this.models.map((model) => option(model.id, model.label || model.id));
this.modelOptions.replaceChildren(...this.models.map((model) => option(model.id, model.label || model.id)));
if (selected && !this.models.some((model) => model.id === selected)) {
choices.unshift(option(selected, `${selected} (current selection; not in loaded list)`));
}
if (!selected) choices.unshift(option("", "Choose a model"));
this.modelSelect.replaceChildren(...choices);
this.modelSelect.value = selected;
}
async discoverModels(showStatus = true) {
const result = await this.client.models(true);
this.modelOptions.replaceChildren(...(result.models || []).map((model) => option(model.id, model.label || model.id)));
if (!this.modelInput.value && result.models?.length) this.modelInput.value = result.models[0].id;
if (showStatus) this.toast(`${result.models?.length || 0} models found`, "success");
const request = ++this.modelRequest;
const provider = this.providerSelect.value;
if (provider !== this.settings.provider) {
this.modelStatus.textContent = "Save the provider connection to load its available models.";
return;
}
this.modelStatus.textContent = "Loading available models…";
try {
const result = await this.client.models(showStatus);
if (this.destroyed || request !== this.modelRequest || provider !== this.providerSelect.value) return;
this.models = result.models || [];
this.renderModelOptions();
this.populateReasoning(this.reasoningSelect.value);
this.modelStatus.textContent = this.models.length
? `${this.models.length} models available. Choose a model, then Save connection.`
: "No models returned. Check the connection or sign-in, then Refresh. Current selection kept.";
if (showStatus) this.toast(this.modelStatus.textContent, this.models.length ? "success" : "error");
} catch (error) {
if (this.destroyed || request !== this.modelRequest || provider !== this.providerSelect.value) return;
this.modelStatus.textContent = `Could not load models: ${error.message}. Current selection kept; try Refresh.`;
if (showStatus) this.toast(this.modelStatus.textContent, "error");
}
}
async subscriptionAction(action) {
@@ -632,6 +656,7 @@ export class BeatPromptWriter {
const result = await this.client.subscription(provider, action);
this.root.querySelector('[data-writer-role="subscription-status"]').textContent = result.message || "Status refreshed.";
if (action === "refresh") this.settings.credential = result;
if (action === "refresh") await this.discoverModels();
this.toast(result.message || "Subscription status refreshed");
}
@@ -888,7 +913,6 @@ export class BeatPromptWriter {
this.threadElement.replaceChildren();
this.currentAssistant = null;
this.activeTools.clear();
this.welcomeElement.hidden = messages.length > 0;
for (const message of messages) this.appendPersistedMessage(message);
this.scrollToBottom(true);
}
@@ -909,7 +933,6 @@ export class BeatPromptWriter {
}
createMessage(role, content = "", message = {}) {
this.welcomeElement.hidden = true;
const article = document.createElement("article");
article.className = `flbps-writer-message ${role}`;
const meta = document.createElement("header");
@@ -1229,6 +1252,11 @@ export class BeatPromptWriter {
: this.pendingAttachments;
}
attachments = attachments.map((attachment) => ({ ...attachment }));
const visionAttachments = [...new Map([...attachments, ...this.nodeSettings.moodboards.filter(board => board?.selected)].map(image => [`${image.subfolder}/${image.filename}`, image])).values()];
if (visionAttachments.length > MAX_CHAT_ATTACHMENTS) {
this.showError(`Use at most ${MAX_CHAT_ATTACHMENTS} images total across chat attachments and checked moodboards.`);
return;
}
if (!text && !attachments.length) return;
if (!this.status?.configured) {
this.showView("settings");
@@ -1247,6 +1275,13 @@ export class BeatPromptWriter {
return;
}
this.currentDocument = document;
this.editor.serialize();
document.sectionIds = Object.fromEntries(document.allowed_indices.map(index => [index, this.editor.clips[index].sectionId]));
document.referenceSelections = Object.fromEntries(document.allowed_indices.map(index => [index, structuredClone(this.editor.clips[index].references)]));
const referenceContext = {
assets: Object.entries(this.editor.referenceAssets || {}).map(([id, asset]) => ({ id, kind: asset.kind, label: asset.label || asset.filename })),
sections: document.allowed_indices.map(index => ({ index, ...this.editor.clips[index].references })),
};
if (!editMessageId) {
this.composer.value = "";
this.pendingAttachments = [];
@@ -1280,10 +1315,11 @@ export class BeatPromptWriter {
conversation_id: this.conversationId,
edit_message_id: editMessageId,
message: text,
attachments,
attachments: visionAttachments,
reference_assets: referenceContext.assets.map(asset => asset.id),
reasoning_effort: this.reasoningComposer.value,
guide_mode: this.nodeSettings.guideMode,
writer_context: this.nodeSettings.context,
writer_context: this.nodeSettings.context + "\nRead-only section reference library (Picture and Video tags each start at 1 per section, ordered within each media type; selections replace defaults; Audio numbering also includes paired video sound and the timeline song): " + JSON.stringify(referenceContext) + "\nMoodboards selected for storyboard generation: " + this.nodeSettings.moodboards.map((board, index) => board?.selected ? `${index+1}: ${board.label}; role: ${board.role}` : "").filter(Boolean).join("; "),
...document,
}, (event) => this.handleRunEvent(event));
} catch (error) {
@@ -1453,6 +1489,10 @@ export class BeatPromptWriter {
});
this.setStatus("Stopped / no prompt changes were applied", "ready");
} else if (event.type === "run_finished") {
if (event.assistantMessage?.metadata?.storyboards?.length) this.generateStoryboards(event.assistantMessage.metadata.storyboards, event.assistantMessage.metadata.updates, event.assistantMessage.id);
const storyboardIndices = new Set((event.assistantMessage?.metadata?.storyboards || []).map(action => action.index));
const assignments = (event.assistantMessage?.metadata?.reference_assignments || []).filter(action => !storyboardIndices.has(action.index));
if (assignments.length) this.applyReferenceAssignments(assignments, event.assistantMessage.metadata.updates);
this.finishAssistantMessage();
if (!["applied", "complete", "no_changes", "error", "stopped"].includes(this.writerActivity.phase)) {
this.updateWriterActivity("complete", "Response complete", { targetIndices: [] });
@@ -1545,6 +1585,7 @@ export class BeatPromptWriter {
}
destroy() {
this.disposeStoryboards?.();
this.destroyed = true;
this.client.detach();
this.editor.clearWriterActivity?.();
@@ -0,0 +1,69 @@
import { app } from "../../../../scripts/app.js";
function removeReferenceInstruction(graph) {
const removedSlots = new Map();
const removedLinks = new Set();
for (const node of graph.nodes ?? []) {
if (node.type !== "FL_KreaReference") continue;
if (node.widgets_values?.length === 9) node.widgets_values.splice(2, 1);
if (node.widgets_values_named) delete node.widgets_values_named.instruction;
const slot = node.inputs?.findIndex(input => input.name === "instruction") ?? -1;
if (slot < 0) continue;
const [input] = node.inputs.splice(slot, 1);
removedSlots.set(String(node.id), slot);
if (input.link != null) removedLinks.add(input.link);
}
if (removedSlots.size) {
graph.links = (graph.links ?? []).filter(link => {
const array = Array.isArray(link);
const id = array ? link[0] : link.id;
if (removedLinks.has(id)) return false;
const slot = removedSlots.get(String(array ? link[3] : link.target_id));
if (slot !== undefined) {
if (array && link[4] > slot) link[4]--;
else if (!array && link.target_slot > slot) link.target_slot--;
}
return true;
});
for (const node of graph.nodes ?? []) {
for (const output of node.outputs ?? []) {
if (output.links) output.links = output.links.filter(id => !removedLinks.has(id));
}
}
}
for (const subgraph of graph.definitions?.subgraphs ?? []) removeReferenceInstruction(subgraph);
}
app.registerExtension({
name: "ComfyUI.FL_KreaReference",
beforeConfigureGraph: removeReferenceInstruction,
nodeCreated(node) {
if (node.constructor?.comfyClass !== "FL_KreaReferenceGuider") return;
const mode = node.widgets.find(widget => widget.name === "blend_mode");
const amount = node.widgets.find(widget => widget.name === "average_amount");
const update = () => {
const connectedMode = node.inputs?.some(input => input.name === "blend_mode" && input.link != null);
amount.disabled = mode.value !== "average" && !connectedMode;
node.setDirtyCanvas(true);
};
const callback = mode.callback;
mode.callback = function () {
const result = callback?.apply(this, arguments);
update();
return result;
};
const onConfigure = node.onConfigure;
node.onConfigure = function () {
const result = onConfigure?.apply(this, arguments);
update();
return result;
};
const onConnectionsChange = node.onConnectionsChange;
node.onConnectionsChange = function () {
const result = onConnectionsChange?.apply(this, arguments);
update();
return result;
};
update();
},
});
+66
View File
@@ -0,0 +1,66 @@
import { app } from "../../../../scripts/app.js";
import { api } from "../../../../scripts/api.js";
app.registerExtension({
name: "ComfyUI.FL_KsamplerSEG.Progress",
nodeCreated(node) {
if (!["FL_KsamplerSEG", "FL_KsamplerSEGAdvanced"].includes(node.comfyClass || node.type)) return;
const element = document.createElement("div");
element.style.cssText = "padding:8px;background:#18181b;color:#fafafa;font:12px sans-serif;box-sizing:border-box;";
const label = document.createElement("div");
label.textContent = "Run to see the active region and full-canvas preview.";
const canvas = document.createElement("canvas");
canvas.width = 300;
canvas.height = 112;
canvas.style.cssText = "display:block;width:100%;height:112px;object-fit:contain;";
const legend = document.createElement("div");
legend.textContent = "White: edit area · cyan: sampler crop · preview below: full canvas";
legend.style.cssText = "font-size:10px;color:#a1a1aa;";
element.append(label, canvas, legend);
const widget = node.addDOMWidget("seg_progress", "seg_progress", element, {
serialize: false, hideOnZoom: false,
getMinHeight: () => 190, getMaxHeight: () => 190,
});
widget.serialize = false;
let mask;
let region;
const draw = (detail) => {
const ctx = canvas.getContext("2d");
const [width, height] = detail.size;
const scale = Math.min(284 / width, 104 / height);
const left = (300 - width * scale) / 2;
const top = (112 - height * scale) / 2;
const [y0, x0, y1, x1] = detail.crop;
ctx.clearRect(0, 0, 300, 112);
ctx.fillStyle = "#09090b";
ctx.fillRect(left, top, width * scale, height * scale);
if (mask) ctx.drawImage(mask, left + x0 * scale, top + y0 * scale, (x1-x0)*scale, (y1-y0)*scale);
ctx.strokeStyle = "#71717a";
ctx.strokeRect(left, top, width * scale, height * scale);
ctx.strokeStyle = "#22d3ee";
ctx.strokeRect(left + x0 * scale, top + y0 * scale, (x1-x0)*scale, (y1-y0)*scale);
};
const onProgress = ({ detail }) => {
if (String(detail.node) !== String(node.id) || node.graph !== app.graph) return;
region = detail.region;
const step = detail.start_step == null ? detail.step : detail.start_step + detail.step;
const steps = detail.start_step == null ? detail.steps : detail.schedule_steps;
label.textContent = `Region ${detail.position}/${detail.count} (mask ${detail.region}) · Step ${step}/${steps}`
+ (detail.state === "complete" ? " · Complete" : detail.state === "stopped" ? " · Stopped" : "");
label.title = detail.start_step == null ? "" : `Window ${detail.start_step}–${detail.end_step}: ${detail.step}/${detail.steps} steps completed`;
if (detail.mask) {
mask = undefined;
const image = new Image();
image.onload = () => {
if (region !== detail.region) return;
mask = image;
draw(detail);
};
image.src = detail.mask;
}
draw(detail);
};
api.addEventListener("fl_seg_sampling", onProgress);
widget.onRemove = () => api.removeEventListener("fl_seg_sampling", onProgress);
},
});
+59 -8
View File
@@ -105,13 +105,14 @@ class SegRegionsPreview {
<div class="flks-seg-canvas-wrap" data-role="canvas-wrap">
<div class="flks-seg-empty">Run the node to preview the tessellation here.</div>
</div>
<div class="flks-seg-footer">cyan = cell boundary · yellow = region index</div>
<div class="flks-seg-footer" data-role="legend">cyan = cell boundary · yellow = region index</div>
`;
this.countEl = this.element.querySelector('[data-role="count"]');
this.canvasWrap = this.element.querySelector('[data-role="canvas-wrap"]');
this.legend = this.element.querySelector('[data-role="legend"]');
}
setPreview(imageDataUri, count, sizeWH) {
setPreview(imageDataUri, count, sizeWH, mode = "overlay", margins = null) {
this.canvasWrap.innerHTML = "";
if (!imageDataUri) return;
const img = document.createElement("img");
@@ -122,6 +123,12 @@ class SegRegionsPreview {
}
this.canvasWrap.appendChild(img);
if (this.countEl) this.countEl.textContent = `${count} regions`;
this.legend.textContent = mode === "coverage_heatmap"
? "red = low coverage · blue = single region · cyan = overlap"
: mode === "sampler_crops"
? "bright = edit mask · dim = surrounding context · labels = crop size"
: "cyan = cell boundary · yellow = region index";
if (margins) this.legend.textContent += ` · band ${margins.overlap}px / feather ${margins.feather}px / context ${margins.context}px`;
}
dispose() {
@@ -132,6 +139,21 @@ class SegRegionsPreview {
}
const INSTANCES = new Map();
const nodeKey = (value) => String(value);
const hiddenWidgets = new WeakMap();
function setMarginWidgetVisible(widget, visible) {
if (!widget) return;
if (!visible && !hiddenWidgets.has(widget)) {
hiddenWidgets.set(widget, { type: widget.type, computeSize: widget.computeSize, hidden: widget.hidden });
widget.type = "converted-widget";
widget.computeSize = () => [0, -4];
widget.hidden = true;
} else if (visible && hiddenWidgets.has(widget)) {
Object.assign(widget, hiddenWidgets.get(widget));
hiddenWidgets.delete(widget);
}
}
app.registerExtension({
name: "ComfyUI.FL_KsamplerSEG_Regions",
@@ -139,6 +161,35 @@ app.registerExtension({
const comfyClass = (node.constructor && node.constructor.comfyClass) || "";
if (comfyClass !== "FL_KsamplerSEG_Regions") return;
const mode = node.widgets.find(w => w.name === "margin_mode");
const overlap = node.widgets.find(w => w.name === "overlap_width_px");
const feather = node.widgets.find(w => w.name === "feather_width_px");
const updateMargins = () => {
const pixels = mode.value === "pixels";
if (pixels) {
feather.options.max = overlap.value / 2;
feather.value = Math.min(feather.value, feather.options.max);
}
for (const name of ["region_overlap_factor", "edge_softness", "context_padding_factor"])
setMarginWidgetVisible(node.widgets.find(w => w.name === name), !pixels);
for (const name of ["overlap_width_px", "feather_width_px", "context_padding_px"])
setMarginWidgetVisible(node.widgets.find(w => w.name === name), pixels);
node.setDirtyCanvas(true, true);
};
const modeCallback = mode.callback;
mode.callback = function (...args) { modeCallback?.apply(this, args); updateMargins(); };
const overlapCallback = overlap.callback;
overlap.callback = function (...args) { overlapCallback?.apply(this, args); updateMargins(); };
const onConfigure = node.onConfigure;
node.onConfigure = function (data) {
const result = onConfigure?.apply(this, arguments);
const saved = data.widgets_values?.[node.widgets.indexOf(mode)];
if (saved !== "pixels" && saved !== "legacy") mode.value = "legacy";
updateMargins();
return result;
};
updateMargins();
const container = document.createElement("div");
container.id = `flks-seg-regions-container-${node.id}`;
container.style.width = "100%";
@@ -161,14 +212,15 @@ app.registerExtension({
setTimeout(() => {
const inst = new SegRegionsPreview({ node, container });
INSTANCES.set(node.id, inst);
INSTANCES.set(nodeKey(node.id), inst);
}, 50);
widget.onRemove = () => {
const inst = INSTANCES.get(node.id);
const key = nodeKey(node.id);
const inst = INSTANCES.get(key);
if (inst) {
inst.dispose();
INSTANCES.delete(node.id);
INSTANCES.delete(key);
}
};
},
@@ -177,8 +229,7 @@ app.registerExtension({
api.addEventListener("fl_seg_regions_preview", (event) => {
const detail = event.detail;
if (!detail) return;
const nodeId = parseInt(detail.node, 10);
const inst = INSTANCES.get(nodeId);
const inst = INSTANCES.get(nodeKey(detail.node));
if (!inst) return;
inst.setPreview(detail.image, detail.count, detail.size);
inst.setPreview(detail.image, detail.count, detail.size, detail.mode, detail.margins);
});
+161
View File
@@ -0,0 +1,161 @@
import { app } from "../../../../scripts/app.js";
import { api } from "../../../../scripts/api.js";
import { createAudioMappingPanel } from "./scan_audio_mapping.js";
import { createPrevis, EFFECT_GROUPS } from "./scan_previs.js";
import { createScanControls } from "./scan_controls.js";
export function previewFrame(time, fps, count) {
return Math.max(0, Math.min(count - 1, Math.floor(time * fps + 0.0001)));
}
export function frameSeekTime(frame, fps) {
return (frame + 0.5) / fps;
}
app.registerExtension({
name: "Fill.InteractiveScanFX",
async beforeRegisterNodeDef(nodeType, nodeData) {
if (nodeData.name !== "FL_InteractiveScanFX") return;
const created = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
created?.apply(this, arguments);
const node = this;
const settings = node.widgets.find(w => w.name === "advanced_settings");
const inputOptions=nodeData.input.required.advanced_settings[1];
const defaults=JSON.parse(inputOptions.default);
const choices=inputOptions.scan_choices;
settings.hidden = true;
settings.type = "converted-widget";
settings.computeSize = () => [0, 0];
settings.computedHeight = 0;
if (settings.element) settings.element.style.display = "none";
const root = document.createElement("div");
root.className = "fl-interactive-scan";
root.style.cssText = "background:#141920;color:#dce4ef;padding:10px;border-radius:8px;font:12px system-ui;overflow:auto;box-sizing:border-box;height:100%";
const layout=document.createElement("div"),visuals=document.createElement("div"),controls=document.createElement("div"),rendered=document.createElement("div"),sources=document.createElement("div");
layout.style.cssText="display:grid;grid-template-columns:minmax(260px,1fr) minmax(380px,1.1fr);gap:10px;height:100%;min-height:0";
visuals.style.cssText="min-width:0;overflow:auto";controls.style.cssText="min-width:0;overflow:auto;padding-right:6px";
sources.style.cssText="display:flex;gap:6px;margin-bottom:10px";
const readSettings=()=>({...defaults,...JSON.parse(settings.value),...Object.fromEntries(node.widgets.filter(w=>Object.values(EFFECT_GROUPS).flat().includes(w.name)).map(w=>[w.name,w.value]))});
const previs=createPrevis(readSettings);
const tabs = document.createElement("div");
tabs.style.cssText = "display:flex;gap:5px;margin-bottom:7px";
const canvas = document.createElement("canvas");
canvas.width = 320; canvas.height = 320;
canvas.style.cssText = "width:100%;max-height:360px;object-fit:contain;background:#080b10;display:block";
const video = document.createElement("video");
video.muted = true; video.playsInline = true; video.preload = "metadata";
const transport = document.createElement("div");
transport.style.cssText = "display:flex;gap:6px;align-items:center;margin:8px 0";
const button = (label, parent, action) => {
const b = document.createElement("button"); b.textContent = label;
b.style.cssText = "background:#293345;color:#e9f1ff;border:1px solid #46536c;border-radius:4px;padding:5px 8px;cursor:pointer";
b.onclick = action; parent.append(b); return b;
};
function setSource(value){previs.setActive(value);rendered.hidden=value;liveButton.style.background=value?"#345d91":"#293345";renderButton.style.background=value?"#293345":"#345d91";if(value)video.pause();}
const liveButton=button("Live demo",sources,()=>setSource(true));
const renderButton=button("Last render",sources,()=>setSource(false));
let view = 0, data = null, frameCallback = null, displayedTime = 0, disposed = false;
const status = document.createElement("div");
const progressBox=document.createElement("div");
progressBox.hidden=true;
const progressLabel=document.createElement("div"),progressBar=document.createElement("progress");
progressLabel.setAttribute("aria-live","polite");
progressBar.setAttribute("aria-label","Current render stage progress");
progressBar.style.cssText="width:100%;height:12px;accent-color:#68b8ff";
progressBox.append(progressLabel,progressBar);
let stage="Preparing",frameUpdates=false;
function showProgress(value,total){
progressBox.hidden=false;progressBar.max=Math.max(1,total);progressBar.value=value;
progressLabel.textContent=`${stage} · ${Math.round(100*value/Math.max(1,total))}% (${value}/${total})`;
}
const caption = document.createElement("div");
caption.style.cssText = "color:#aab8cc;margin:6px 0;font-size:11px";
const labels = ["Final", "Surface", "Mask", "Debug", "Depth"];
const descriptions = ["Finished effect", "Projected voxel surface · before camera accents", "Reveal alpha · excludes cursor graphics", "Planned cursor trajectories · frame and shot", "Projected depth · before camera accents"];
let mappingPanel;
const tabButtons = labels.map((label,i) => button(label,tabs,()=>{view=i;draw();}));
const scrub = document.createElement("input");scrub.type="range";scrub.min="0";scrub.max="0";scrub.step="1";scrub.style.width="100%";scrub.setAttribute("aria-label","Preview frame");
const readout = document.createElement("span");readout.style.whiteSpace="nowrap";
const meters = document.createElement("div");meters.style.cssText="display:grid;grid-template-columns:repeat(3,1fr);gap:8px;margin:8px 0";
const bars = ["Kick", "Snare", "Hat"].map(label=>{
const cell=document.createElement("label");cell.textContent=label;
const meter=document.createElement("meter");meter.min=0;meter.max=1;meter.value=0;meter.style.width="100%";cell.append(meter);meters.append(cell);return meter;
});
function draw() {
tabButtons.forEach((b,i)=>{b.style.background=i===view?"#345d91":"#293345";b.disabled=!!data&&i>=(data.views?.length??4);});
caption.textContent=descriptions[view];
if (!data) {status.textContent="Run the node to generate previews.";return;}
const frame=previewFrame(displayedTime,data.fps,data.frames);
scrub.value=String(frame);readout.textContent=`${frame+1} / ${data.frames}`;
bars.forEach((b,i)=>b.value=data.envelopes[i][frame]??0);
const columns=data.columns??2;
if(video.readyState>=2)canvas.getContext("2d").drawImage(video,(view%columns)*data.width,Math.floor(view/columns)*data.height,data.width,data.height,0,0,canvas.width,canvas.height);
play.textContent=video.paused?"Play":"Pause";
mappingPanel?.update(data,frame);
controlPanel?.update(data,frame);
}
function tick(now, metadata){frameCallback=null;if(disposed)return;displayedTime=metadata.mediaTime;draw();if(!video.paused)frameCallback=video.requestVideoFrameCallback(tick);}
const play=button("Play",transport,async()=>{
if(!data)return;
if(video.paused){if(video.ended)video.currentTime=0;try{await video.play();}catch(error){status.textContent=error.message;return;}if(frameCallback===null)frameCallback=video.requestVideoFrameCallback(tick);}else video.pause();draw();
});
button("Stop",transport,()=>{video.pause();video.currentTime=0;displayedTime=0;draw();});
transport.append(scrub,readout);
scrub.oninput=()=>{if(data){video.pause();video.currentTime=frameSeekTime(Number(scrub.value),data.fps);}};
video.onloadeddata=video.onseeked=()=>{displayedTime=video.currentTime;draw();};video.onended=draw;
video.onerror=()=>{status.textContent="Preview unavailable. Run again to recreate the temporary preview.";};
let controlPanel;
function populateSettings(){
settings.value=JSON.stringify({...defaults,...JSON.parse(settings.value)});
controlPanel?.rebuild();mappingPanel?.rebuild();
previs.redraw();
}
mappingPanel=createAudioMappingPanel({settings,targets:inputOptions.scan_mapping_targets,
onChange:()=>{node.graph?.setDirtyCanvas(true,true);controlPanel?.rebuild();status.textContent="Mapping changed · run again to update rendered values.";},
onSeek:frame=>{if(data){video.pause();video.currentTime=frameSeekTime(frame,data.fps);}}});
controlPanel=createScanControls({node,settings,options:{...inputOptions,widget_specs:Object.fromEntries(Object.entries(nodeData.input.required).map(([k,v])=>[k,v[1]]))},mappingPanel,
onChange:()=>{previs.redraw();status.textContent="Settings changed · run again to update rendered values.";},
onSolo:key=>{setSource(true);previs.select(key);}});
rendered.append(tabs,canvas,caption,transport,meters,status);
visuals.append(progressBox,sources,previs.element,rendered);
visuals.append(mappingPanel.element);controls.append(controlPanel.element);layout.append(visuals,controls);root.append(layout);setSource(true);
node.addDOMWidget("scan_preview","fl-scan-preview",root,{serialize:false,hideOnZoom:false,getMinHeight:()=>540});
function load(preview){
if(!preview)return;
data=preview;node.properties.fl_scan_preview=preview;
video.pause();displayedTime=0;video.src=api.apiURL(`/view?${new URLSearchParams({filename:preview.filename,subfolder:preview.subfolder,type:preview.type})}`);
canvas.width=preview.width;canvas.height=preview.height;scrub.max=String(preview.frames-1);
status.textContent=`${preview.frames} frames · ${preview.cuts} cuts · ${preview.events} gestures · silent preview from last run`;
draw();
}
const executed=node.onExecuted;
node.onExecuted=function(message){executed?.apply(this,arguments);load(message.fl_interactive_scan?.[0]);};
const configured=node.onConfigure;
node.onConfigure=function(){configured?.apply(this,arguments);populateSettings();load(node.properties.fl_scan_preview);};
const connectionsChanged=node.onConnectionsChange;
node.onConnectionsChange=function(){connectionsChanged?.apply(this,arguments);queueMicrotask(()=>{if(!disposed)controlPanel?.rebuild();});};
const removed=node.onRemoved;
const stageProgress=event=>{
const d=event.detail;if(String(d.node)!==String(node.id))return;
stage=d.stage;frameUpdates=d.frame_updates;showProgress(d.value,d.max);
};
const frameProgress=event=>{
const d=event.detail;
if(frameUpdates&&String(d.node)===String(node.id))showProgress(d.value,d.max);
};
const failed=event=>{
if(String(event.detail.node_id)!==String(node.id))return;
frameUpdates=false;progressLabel.textContent="Render stopped · check the execution error";
};
api.addEventListener("fl_scan_progress",stageProgress);
api.addEventListener("progress",frameProgress);
api.addEventListener("execution_error",failed);
api.addEventListener("execution_interrupted",failed);
const executing=event=>{if(String(event.detail?.node??event.detail)===String(node.id))status.textContent="Rendering effect and diagnostic previews…";};
api.addEventListener("executing",executing);
node.onRemoved=function(){disposed=true;previs.dispose();api.removeEventListener("executing",executing);api.removeEventListener("fl_scan_progress",stageProgress);api.removeEventListener("progress",frameProgress);api.removeEventListener("execution_error",failed);api.removeEventListener("execution_interrupted",failed);if(frameCallback!==null)video.cancelVideoFrameCallback(frameCallback);video.pause();video.removeAttribute("src");video.load();root.remove();removed?.apply(this,arguments);};
populateSettings();draw();node.setSize([1000,950]);
};
},
});
+139
View File
@@ -0,0 +1,139 @@
import {app} from "../../../../scripts/app.js";
import {api} from "../../../../scripts/api.js";
import {CAMERA_PRESETS, demoLayers, drawScene, viewport} from "./parallax_preview.js";
import {createDepthPreview} from "./parallax_depth_preview.js";
const CONTROLS = ["motion","travel_x","travel_y","push_in","background_depth","overscan","device","layer_fit","relief_scope","relief_strength","relief_anchor","depth_invert","depth_smoothing"];
const style = document.createElement("style");
style.textContent = `
.fl-parallax-editor{height:100%;box-sizing:border-box;overflow:auto;padding:12px;background:#111a26;color:#e6edf7;font:12px system-ui;border:1px solid #334459;border-radius:9px}
.fl-parallax-editor *{box-sizing:border-box}.fl-parallax-editor .pe-top,.fl-parallax-editor .pe-tabs,.fl-parallax-editor .pe-presets{display:flex;gap:6px;align-items:center;flex-wrap:wrap}
.fl-parallax-editor .pe-top{justify-content:space-between;margin-bottom:10px}.fl-parallax-editor .pe-body{display:grid;grid-template-columns:minmax(0,1fr) 235px;gap:12px}
.fl-parallax-editor button,.fl-parallax-editor select,.fl-parallax-editor input[type=number]{background:#223247;color:#e6edf7;border:1px solid #425772;border-radius:5px;padding:5px;font:inherit}
.fl-parallax-editor button{cursor:pointer}.fl-parallax-editor button[aria-pressed=true]{background:#23665f;border-color:#68c6b5}.fl-parallax-editor button:disabled{opacity:.45;cursor:default}
.fl-parallax-editor canvas{display:block;width:100%;background:#0b111c;border:1px solid #34485d;border-radius:6px;touch-action:none}.fl-parallax-editor .pe-scene{cursor:grab;margin:8px 0}
.fl-parallax-editor .pe-pad{height:105px;cursor:crosshair;margin:8px 0}.fl-parallax-editor .pe-hint{font-size:11px;color:#9fb1c7;line-height:1.5;margin:6px 0}
.fl-parallax-editor .pe-field{display:grid;grid-template-columns:minmax(0,1fr) 68px;gap:5px;align-items:center;margin:12px 0}.fl-parallax-editor .pe-field input[type=range]{grid-column:1/-1;width:100%;accent-color:#75cbbb}
.fl-parallax-editor .pe-field input[type=number]{width:68px}.fl-parallax-editor .pe-help{border-radius:50%;padding:0 4px;margin-left:5px;font-size:10px;cursor:help}
.fl-parallax-editor .pe-tabs{border-bottom:1px solid #334459;padding-bottom:8px;margin-bottom:8px}.fl-parallax-editor .pe-warning{color:#f4c184;min-height:30px;font-size:11px;line-height:1.4}
.fl-parallax-editor .pe-status{color:#8fd8c5;font-size:11px;margin-top:8px}.fl-parallax-editor input[type=range]{min-width:0}.fl-parallax-editor [hidden]{display:none!important}
`;
document.head.append(style);
app.registerExtension({
name: "Fill.LayeredParallax.Editor",
async beforeRegisterNodeDef(nodeType, data) {
if (data.name !== "FL_LayeredParallax") return;
const created = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function() {
created?.apply(this,arguments);
const node=this, widgets=Object.fromEntries(node.widgets.map(w=>[w.name,w]));
for (const name of CONTROLS) if (widgets[name]) {
const w=widgets[name];w.type="converted-widget";w.computeSize=()=>[0,-4];w.hidden=true;
if(w.element)w.element.style.display="none";
}
const root=document.createElement("div");root.className="fl-parallax-editor";
root.innerHTML=`<div class="pe-top"><strong>PARALLAX STUDIO</strong><span>Camera & framing · no diffusion</span></div>
<div class="pe-body"><div><div class="pe-tabs"><button data-view="scene" aria-pressed="true">Scene</button><button data-view="depth" aria-pressed="false">Depth order</button></div>
<canvas class="pe-scene" width="480" height="288" aria-label="Parallax preview. Drag to adjust camera travel."></canvas>
<div class="pe-presets"><button data-play aria-label="Play preview">Play</button><input data-phase aria-label="Preview position" type="range" min="0" max="1" step=".001" value="0" style="flex:1"><button data-center>Center</button></div>
<p class="pe-hint" data-source-label></p><select data-source aria-label="Preview source"><option value="actual">Actual layers</option><option value="landscape">Demo · landscape</option><option value="design">Demo · typography</option><option value="product">Demo · product shapes</option></select>
<p class="pe-hint">Drag the scene to set travel. Preview uses small first-frame images and the render's camera math; it is not a final-quality render.</p><p class="pe-warning" role="status"></p><div class="pe-status"></div></div>
<div><div class="pe-tabs"><button data-tab="camera" aria-pressed="true">Camera</button><button data-tab="framing" aria-pressed="false">Framing</button></div>
<div data-page="camera"><div class="pe-presets" data-presets></div><p class="pe-hint">Presets change motion, not the image's style.</p><div class="pe-presets" data-motions></div>
<canvas class="pe-pad" width="235" height="105" aria-label="Camera direction control. Drag the point to set horizontal and vertical travel."></canvas><div data-camera-fields></div></div>
<div data-page="framing" hidden><div data-framing-fields></div><div class="pe-field"><label>Cutout fit</label><select data-select="layer_fit"><option>cover</option><option>contain</option></select></div><p class="pe-hint">Contain keeps the full cutout when aspect ratios differ. Cover fills the frame. Background edges extend existing pixels; no new scenery is generated.</p>
<div class="pe-field"><label>Render device</label><select data-select="device"><option>auto</option><option>cpu</option></select></div><p class="pe-hint">Layer depths, placement and visibility come from the connected layer controls. Width, height and frame count remain above this editor.</p></div></div></div>`;
node.addDOMWidget("parallax_editor","div",root,{serialize:false});
const reliefPreview=createDepthPreview();
const tabs=root.querySelector('[data-tab="camera"]').parentElement;
const reliefTab=document.createElement('button');reliefTab.dataset.tab='relief';reliefTab.textContent='Relief';tabs.append(reliefTab);
const reliefPage=document.createElement('div');reliefPage.dataset.page='relief';reliefPage.hidden=true;
reliefPage.innerHTML='<div class="pe-field"><label>Apply to</label><select data-select="relief_scope"><option>off</option><option>background</option><option>background + artwork</option></select></div><div data-relief-fields></div><label><input type="checkbox" data-invert> Invert depth (black is near)</label><p class="pe-hint">Text stays flat. Depth batches must match the current layer stack. Strength and anchor preview live; smoothing requires Run. Preview uses a small depth grid. Relief cannot reveal missing surfaces or change layer draw order.</p>';
tabs.parentElement.append(reliefPage);
const scene=root.querySelector(".pe-scene"),pad=root.querySelector(".pe-pad"),phaseInput=root.querySelector("[data-phase]"),play=root.querySelector("[data-play]"),status=root.querySelector(".pe-status"),warning=root.querySelector(".pe-warning"),sourceSelect=root.querySelector("[data-source]"),sourceLabel=root.querySelector("[data-source-label]");
let phase=0,centered=false,playing=false,visible=false,disposed=false,raf=0,last=0,drawn=0,view="scene",actual=[],demo=demoLayers("landscape"),loadVersion=0;
const linked=name=>node.inputs?.some(i=>i.name===name&&i.link!=null);
const settings=()=>Object.fromEntries(["width","height",...CONTROLS].map(k=>[k,(linked(k)?node.properties.parallax_settings?.[k]:undefined)??widgets[k]?.value??(k==="layer_fit"?"cover":0)]));
const plates=()=>sourceSelect.value==="actual"&&actual.length?actual:demo;
function write(name,value) {
if(linked(name)){status.textContent=`${name} is driven by a connected input.`;return;}
if(!widgets[name])return;
if(name==="background_depth")value=Math.min(100,Math.max(value,...plates().filter(p=>!p.background).map(p=>p.depth+.25)));
widgets[name].value=value;node.graph?.change();node.graph?.setDirtyCanvas(true,true);refresh();invalidate();
}
function refresh() {
const s=settings();
root.querySelectorAll("[data-setting]").forEach(e=>{e.value=s[e.dataset.setting];e.disabled=linked(e.dataset.setting);});
root.querySelectorAll("[data-select]").forEach(e=>{e.value=s[e.dataset.select];e.disabled=linked(e.dataset.select);});
root.querySelector('[data-invert]').checked=!!s.depth_invert;root.querySelector('[data-invert]').disabled=linked('depth_invert');
root.querySelectorAll("[data-motion]").forEach(e=>{e.setAttribute("aria-pressed",String(e.dataset.motion===s.motion));e.disabled=linked("motion");});
root.querySelectorAll("[data-preset]").forEach(e=>e.setAttribute("aria-pressed",String(Object.entries(CAMERA_PRESETS[e.dataset.preset]).every(([k,v])=>s[k]===v))));
const nearest=Math.min(...plates().filter(p=>!p.background).map(p=>p.depth),s.background_depth);
const farthest=Math.max(...plates().filter(p=>!p.background).map(p=>p.depth),0);
root.querySelectorAll('[data-setting="background_depth"]').forEach(e=>e.min=Math.min(100,Math.max(.25,farthest+.25)));
warning.textContent=s.background_depth<=farthest?"Background depth must be greater than every cutout depth.":Math.abs(s.push_in)>=nearest?"Reduce push-in: the camera must stay in front of every layer.":Math.hypot(s.travel_x,s.travel_y)/nearest>.08?"Strong motion can reveal missing edges or hidden regions. Reduce travel or prepare wider layers.":"";
if(s.relief_scope!=="off"&&s.relief_strength>0){
if(!plates().some(p=>p.relief))warning.textContent="Run with depth maps once to enable the relief preview. Showing flat camera motion.";
else if(!reliefPreview)warning.textContent="WebGL unavailable: preview shows flat layers; depth relief still applies on Run.";
}
}
const ranges=[
["travel_x","Horizontal travel",-2,2,.005,"Camera movement in output widths at depth 1. Nearer layers move more.","camera"],
["travel_y","Vertical travel",-2,2,.005,"Camera movement in output heights at depth 1.","camera"],
["push_in","Push / pull",-.2,.2,.005,"Forward travel in depth units. Positive moves toward the layers.","camera"],
["background_depth","Background depth",.25,100,.25,"Must be farther away than every foreground layer. Does not change their depth order.","framing"],
["overscan","Background coverage",1,2,.01,"Extra background scale. More coverage reduces visible border extension.","framing"],
["relief_strength","Relief strength",0,.5,.01,"Variation within each layer. Zero keeps the original flat-plane rendering.","relief"],
["relief_anchor","Anchor depth",0,1,.01,"Depth-map value that follows the original flat plane. White is near unless inverted.","relief"],
["depth_smoothing","Depth smoothing",0,16,1,"Smoothing radius in depth-map pixels. Run to apply; cached depth inference is reused.","relief"],
];
for(const [name,label,min,max,step,help,page] of ranges){
const field=document.createElement("div");field.className="pe-field";
const title=document.createElement("label");title.textContent=label;
const tip=document.createElement("button");tip.textContent="?";tip.className="pe-help";tip.title=help;tip.setAttribute("aria-label",help);tip.onclick=()=>status.textContent=help;title.append(tip);field.append(title);
for(const type of ["number","range"]){const input=document.createElement("input");input.type=type;input.min=min;input.max=max;input.step=step;input.dataset.setting=name;input.setAttribute("aria-label",`${label} ${type}`);input.title=help;
input.addEventListener(type==="range"?"input":"change",()=>{const v=Number(input.value);if(input.value!==""&&Number.isFinite(v))write(name,Math.max(min,Math.min(max,v)));else refresh();});field.append(input);}
root.querySelector(`[data-${page}-fields]`).append(field);
}
for(const [label,preset] of Object.entries(CAMERA_PRESETS)){
const b=document.createElement("button");b.textContent=label;b.dataset.preset=label;b.onclick=()=>{for(const [k,v] of Object.entries(preset))write(k,v);status.textContent=Object.keys(preset).some(linked)?"Preset applied to unconnected controls; connected inputs stay unchanged.":`${label} motion applied. Run to export.`;};root.querySelector("[data-presets]").append(b);
}
for(const motion of ["loop","glide","bursts","locked"]){const b=document.createElement("button");b.textContent=motion;b.dataset.motion=motion;b.onclick=()=>write("motion",motion);root.querySelector("[data-motions]").append(b);}
root.querySelectorAll("[data-select]").forEach(e=>e.onchange=()=>write(e.dataset.select,e.value));
root.querySelector('[data-invert]').onchange=e=>write('depth_invert',e.target.checked);
root.querySelectorAll("[data-tab]").forEach(b=>b.onclick=()=>{root.querySelectorAll("[data-tab]").forEach(e=>e.setAttribute("aria-pressed",String(e===b)));root.querySelectorAll("[data-page]").forEach(e=>e.hidden=e.dataset.page!==b.dataset.tab);});
root.querySelectorAll("[data-view]").forEach(b=>b.onclick=()=>{view=b.dataset.view;if(view==="depth")stop();root.querySelectorAll("[data-view]").forEach(e=>e.setAttribute("aria-pressed",String(e===b)));invalidate();});
function drawPad(s){const c=pad.getContext("2d"),w=pad.width,h=pad.height;c.clearRect(0,0,w,h);c.strokeStyle="#3b5068";c.beginPath();c.moveTo(w/2,0);c.lineTo(w/2,h);c.moveTo(0,h/2);c.lineTo(w,h/2);c.stroke();
const x=w/2+Math.max(-.5,Math.min(.5,s.travel_x))*w,y=h/2+Math.max(-.5,Math.min(.5,s.travel_y))*h;c.strokeStyle="#85d5c4";c.beginPath();c.moveTo(w/2,h/2);c.lineTo(x,y);c.stroke();c.fillStyle="#85d5c4";c.beginPath();c.arc(x,y,5,0,7);c.fill();c.font="10px system-ui";c.fillStyle="#b7c9da";c.fillText("CAMERA XY · ±0.5",6,14);}
function paint(now){raf=0;if(disposed||!visible||document.hidden)return;
if(playing){if(last)phase=(phase+(now-last)/6000)%1;last=now;}else last=0;
if(!playing||now-drawn>=50){const s=settings(),height=view==="depth"?Math.max(360,plates().length*26+50):288;if(scene.height!==height)scene.height=height;drawScene(scene,plates(),centered?{...s,motion:"locked"}:s,phase,view==="depth",reliefPreview);drawPad(s);phaseInput.value=phase;drawn=now;}
if(playing)raf=requestAnimationFrame(paint);
}
function invalidate(){if(!raf&&visible&&!document.hidden&&!disposed)raf=requestAnimationFrame(paint);}
function stop(){playing=false;last=0;play.textContent="Play";play.setAttribute("aria-label","Play preview");if(raf)cancelAnimationFrame(raf);raf=0;}
play.onclick=()=>{centered=false;playing=!playing;play.textContent=playing?"Pause":"Play";play.setAttribute("aria-label",playing?"Pause preview":"Play preview");last=0;invalidate();};
phaseInput.oninput=()=>{stop();centered=false;phase=Number(phaseInput.value);invalidate();};
root.querySelector("[data-center]").onclick=()=>{stop();centered=true;phase=0;invalidate();};
function drag(canvas,move){let active=false;canvas.onpointerdown=e=>{if(view==="depth"&&canvas===scene)return;active=true;canvas.setPointerCapture(e.pointerId);stop();centered=false;if(settings().motion==="locked")write("motion","loop");phase=settings().motion==="loop"?.25:1;move(e,true);invalidate();e.stopPropagation();};canvas.onpointermove=e=>{if(active){move(e,false);e.stopPropagation();}};canvas.onpointerup=canvas.onpointercancel=e=>{active=false;if(canvas.hasPointerCapture(e.pointerId))canvas.releasePointerCapture(e.pointerId);};}
drag(pad,e=>{const r=pad.getBoundingClientRect();write("travel_x",Math.max(-.5,Math.min(.5,(e.clientX-r.left)/r.width-.5)));write("travel_y",Math.max(-.5,Math.min(.5,(e.clientY-r.top)/r.height-.5)));});
let origin;
drag(scene,(e,start)=>{const s=settings();if(start){origin={x:e.clientX,y:e.clientY,tx:s.travel_x,ty:s.travel_y};return;}const r=scene.getBoundingClientRect(),v=viewport(r.width,r.height,s.width/s.height),depth=Math.min(...plates().filter(p=>!p.background).map(p=>p.depth),s.background_depth);write("travel_x",Math.max(-2,Math.min(2,origin.tx-(e.clientX-origin.x)/v.w*depth)));write("travel_y",Math.max(-2,Math.min(2,origin.ty-(e.clientY-origin.y)/v.h*depth)));});
function sourceChanged(){if(sourceSelect.value!=="actual")demo=demoLayers(sourceSelect.value);sourceLabel.textContent=sourceSelect.value==="actual"&&actual.length?`${actual.length} actual layers · first-frame preview · upstream changes need Run`:"DEMO GEOMETRY · not generated output";refresh();invalidate();}
sourceSelect.onchange=sourceChanged;
async function load(rows){reliefPreview?.clear();const version=++loadVersion;if(!rows?.length){sourceChanged();return;}const loaded=await Promise.all(rows.map(p=>new Promise(resolve=>{const bitmap=new Image();bitmap.decoding="async";bitmap.onload=()=>resolve({...p,bitmap});bitmap.onerror=()=>resolve(null);bitmap.src=api.apiURL(`/view?${new URLSearchParams(p.image)}`);})));if(disposed||version!==loadVersion)return;actual=loaded.every(Boolean)?loaded.sort((a,b)=>a.background?-1:b.background?1:b.depth-a.depth):[];sourceChanged();if(!actual.length)status.textContent="Source previews unavailable; showing demo. Run once to rebuild thumbnails.";}
function restore(){const rows=node.properties.parallax_layers;if(rows){load(rows);return;}const link=node.inputs?.find(i=>i.name==="layer_stack")?.link;const source=node.graph?.getNodeById(node.graph?.links[link]?.origin_id);const review=source?.properties.poster_review?.poster_layers;
if(review)load(review.filter(r=>r.kind==="background"||r.visible).map(r=>({...r,image:r.thumbnail,width:widgets.width.value,height:widgets.height.value,background:r.kind==="background",opacity:1})));else sourceChanged();}
const observer=new IntersectionObserver(entries=>{visible=entries[0].isIntersecting;if(!visible){if(raf)cancelAnimationFrame(raf);raf=0;last=0;}else invalidate();});observer.observe(root);
const visibility=()=>{if(document.hidden){if(raf)cancelAnimationFrame(raf);raf=0;last=0;}else invalidate();};document.addEventListener("visibilitychange",visibility);
const progress=e=>{if(String(e.detail.node)===String(node.id))status.textContent=`Rendering ${Math.round(e.detail.value/e.detail.max*100)}%`;};api.addEventListener("progress",progress);
for(const w of Object.values(widgets)){const callback=w.callback;w.callback=function(){const result=callback?.apply(this,arguments);refresh();invalidate();return result;};}
const executed=node.onExecuted,configured=node.onConfigure,removed=node.onRemoved;
node.onExecuted=function(message){executed?.apply(this,arguments);if(message.parallax_settings)node.properties.parallax_settings=message.parallax_settings[0];if(message.parallax_layers){node.properties.parallax_layers=message.parallax_layers;load(message.parallax_layers);status.textContent="Rendered · camera edits preview immediately; Run exports them.";}};
node.onConfigure=function(){configured?.apply(this,arguments);restore();refresh();};
node.onRemoved=function(){disposed=true;stop();reliefPreview?.dispose();observer.disconnect();document.removeEventListener("visibilitychange",visibility);api.removeEventListener("progress",progress);root.remove();removed?.apply(this,arguments);};
queueMicrotask(()=>{if(!disposed){restore();refresh();}});node.setSize([850,750]);
};
},
});
+184
View File
@@ -0,0 +1,184 @@
import { app } from "../../../../scripts/app.js";
import { api } from "../../../../scripts/api.js";
import { ownsPosterEvent, posterLayerId, replacePreviewUrl } from "./poster_preview_events.js";
app.registerExtension({
name: "Fill.PosterLayers",
async beforeRegisterNodeDef(nodeType, nodeData) {
if (nodeData.name !== "FL_PosterLayers" && nodeData.name !== "FL_PosterLayerPlanner") return;
const created = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
created?.apply(this, arguments);
const node = this;
if (nodeData.name === "FL_PosterLayerPlanner") {
const executed = node.onExecuted;
node.onExecuted = function (message) {
executed?.apply(this, arguments);
if (message.poster_plan?.[0]) node.properties.poster_plan = message.poster_plan[0];
};
node.addWidget("button", "Use last plan in Manual mode", null, () => {
if (!node.properties.poster_plan) return;
node.widgets.find(w => w.name === "manual_plan").value = JSON.stringify(node.properties.poster_plan, null, 2);
node.widgets.find(w => w.name === "mode").value = "manual";
app.graph.change();
});
return;
}
const widget = node.widgets.find(w => w.name === "overrides");
widget.type = "converted-widget";
widget.computeSize = () => [0, -4];
widget.hidden = true;
if (widget.element) widget.element.style.display = "none";
const root = document.createElement("div");
root.style.cssText = "height:100%;overflow:auto;background:#141820;color:#e9edf5;padding:12px;font:12px system-ui;box-sizing:border-box;border-radius:8px";
const status = document.createElement("div"), cards = document.createElement("div");
status.textContent = "Run to extract the planned layers. Layout edits apply on the next Run.";
status.style.cssText = "padding:0 0 12px;color:#a7b8d2";
cards.style.cssText = "display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px";
root.append(status, cards);
const live=document.createElement("section"),liveLabel=document.createElement("div"),liveImage=document.createElement("img"),liveProgress=document.createElement("progress");
live.hidden=true;live.style.cssText="background:#101722;border:1px solid #425772;border-radius:6px;padding:10px;margin-bottom:12px";
liveLabel.setAttribute("aria-live","polite");liveImage.alt="Current layer preview";liveImage.decoding="async";
liveImage.style.cssText="display:block;width:100%;height:240px;object-fit:contain;margin:8px 0;background:repeating-conic-gradient(#343c48 0% 25%,#252c37 0% 50%) 50%/20px 20px";
liveImage.hidden=true;liveProgress.max=1;liveProgress.value=0;liveProgress.style.cssText="width:100%;height:10px;accent-color:#77c8bb";
liveProgress.setAttribute("aria-label","Current layer sampling progress");live.append(liveLabel,liveImage,liveProgress);root.insertBefore(live,cards);
node.addDOMWidget("poster_review", "div", root, { serialize: false });
// This node owns its image display; skip Comfy's duplicate canvas image widget.
node.onDrawBackground=function() {};
const removeCanvasPreview=()=>{
const index=node.widgets.findIndex(w=>w.name==="$$canvas-image-preview");
if(index>=0){node.widgets[index].onRemove?.();node.widgets.splice(index,1);}
node.imgs=undefined;
};
let rows = [], key = "", disposed = false;
const previewState={url:null};let currentLayer=null;
const layerImages=new Map();
const belongs=d=>!disposed&&node.graph===app.graph&&ownsPosterEvent(node.id,d);
const read = () => {
const data = JSON.parse(widget.value || "{}");
return data.plan_key === key ? data : { plan_key: key, layers: {} };
};
const edit = (id, field, value) => {
const data = read();
data.layers[id] = { ...data.layers[id], [field]: value };
widget.value = JSON.stringify(data);
app.graph.change();
status.textContent = "Edits pending — Run to apply. Only changed extraction prompts/rerolls need diffusion.";
};
const button = (label, parent, action) => {
const b = document.createElement("button");
b.textContent = label;
b.style.cssText = "background:#303b51;color:#f2f5fa;border:1px solid #50617d;border-radius:4px;padding:5px 8px;cursor:pointer";
b.onclick = action; parent.append(b); return b;
};
function draw() {
cards.replaceChildren();
layerImages.clear();
for (const row of rows) {
const values = { ...row.defaults, ...read().layers[row.id] };
const card = document.createElement("section");
card.style.cssText = "background:#1e2633;border:1px solid #39465d;border-radius:6px;padding:10px;min-width:0";
const heading = document.createElement("strong");
heading.textContent = `${row.name} · ${row.kind}`;
const image = document.createElement("img");
image.src = api.apiURL(`/view?${new URLSearchParams(row.thumbnail)}`);
layerImages.set(row.id,image);
image.loading = "lazy"; image.decoding = "async"; image.alt = row.name;
image.style.cssText = "width:100%;height:160px;object-fit:contain;margin:8px 0;cursor:zoom-in;background:repeating-conic-gradient(#343c48 0% 25%,#252c37 0% 50%) 50%/20px 20px";
image.onclick = () => {
const dialog = document.createElement("dialog"), full = document.createElement("img");
dialog.style.cssText = "background:#18202b;color:white;max-width:85vw;max-height:90vh;border:1px solid #52637a;border-radius:8px";
full.src = api.apiURL(`/view?${new URLSearchParams(row.file)}`);
full.style.cssText = "display:block;max-width:78vw;max-height:77vh;object-fit:contain";
dialog.append(full); button("Close", dialog, () => dialog.close());
dialog.addEventListener("close", () => dialog.remove(), { once: true });
document.body.append(dialog); dialog.showModal();
};
const prompt = document.createElement("textarea");
prompt.value = values.prompt; prompt.rows = 3;
prompt.setAttribute("aria-label", `${row.name} extraction prompt`);
prompt.style.cssText = "box-sizing:border-box;width:100%;resize:vertical;background:#111822;color:#e9edf5;border:1px solid #4c5c74;border-radius:4px;padding:6px";
prompt.onchange = () => { if (prompt.value.trim()) edit(row.id, "prompt", prompt.value.trim()); else prompt.value = values.prompt; };
const fields = document.createElement("div");
fields.style.cssText = "display:grid;grid-template-columns:1fr 1fr;gap:6px;margin:8px 0";
for (const [field, label, min, max, step] of [["depth", "Depth", .25, 11.75, .25], ["scale", "Scale", .1, 4, .01], ["offset_x", "X offset", -2, 2, .01], ["offset_y", "Y offset", -2, 2, .01]]) {
if (row.kind === "background") continue;
const l = document.createElement("label"), input = document.createElement("input");
l.textContent = label + " "; input.type = "number"; input.min = min; input.max = max; input.step = step; input.value = values[field];
input.title = field === "depth" ? "Smaller = in front and faster. Must be less than camera background depth." : "Applied by the compositor; does not rerun diffusion.";
input.style.cssText = "width:70px;background:#111822;color:white;border:1px solid #4c5c74;border-radius:3px";
input.onchange = () => { const v = Number(input.value); if (input.value !== "" && Number.isFinite(v)) { input.value = Math.max(min, Math.min(max, v)); edit(row.id, field, Number(input.value)); } };
l.append(input); fields.append(l);
}
const actions = document.createElement("div");
actions.style.cssText = "display:flex;gap:6px;align-items:center;flex-wrap:wrap";
if (row.kind !== "background") {
const label = document.createElement("label"), visible = document.createElement("input");
visible.type = "checkbox"; visible.checked = values.visible; visible.onchange = () => edit(row.id, "visible", visible.checked);
label.append(visible, " Visible"); actions.append(label);
}
button("Reroll", actions, async () => {
edit(row.id, "revision", (read().layers[row.id]?.revision ?? row.defaults.revision) + 1);
status.textContent = `Queuing ${row.name} reroll…`;
try { await api.queuePrompt(0, await app.graphToPrompt()); status.textContent = "Reroll queued. Other layers can reuse their cache."; }
catch (error) { status.textContent = `Could not queue: ${error.message}`; }
});
button("Reset", actions, () => {
const data = read(); delete data.layers[row.id]; widget.value = JSON.stringify(data); app.graph.change();
status.textContent = "Plan defaults restored — Run to apply.";
draw();
});
const coverage = document.createElement("div");
const incompleteBackground = row.kind === "background" && row.coverage < .98;
coverage.textContent = incompleteBackground ? "Background has missing areas — use a full-frame background prompt." : row.coverage < .001 ? "Nearly empty extraction — check prompt or reroll." : `Visible alpha: ${(row.coverage * 100).toFixed(1)}%`;
coverage.style.cssText = `color:${incompleteBackground ? "#f4c184" : "#a8b5c9"};margin-top:8px`;
card.append(heading, image, prompt, fields, actions, coverage); cards.append(card);
}
}
function load(message) {
if (!message?.poster_layers) return;
rows = message.poster_layers; key = message.poster_plan_key[0];
node.properties.poster_review = { poster_layers: rows, poster_plan_key: [key] };
status.textContent = `${rows.length} layers · click a thumbnail for full-resolution RGBA · Run applies layout edits`;
draw();
}
const executed = node.onExecuted, configured = node.onConfigure, removed = node.onRemoved;
node.onExecuted = function (message) { executed?.apply(this, arguments); load(message); };
node.onConfigure = function () { configured?.apply(this, arguments); removeCanvasPreview(); load(node.properties.poster_review); };
const progress = event => {
const d = event.detail;
if(!belongs(d))return;
const id=posterLayerId(d);
if(!id&&liveProgress.hidden)currentLayer=null;
if(id&&id!==currentLayer){currentLayer=id;replacePreviewUrl(previewState,null);liveImage.removeAttribute("src");liveImage.hidden=true;}
live.hidden=false;liveProgress.hidden=false;liveProgress.max=Math.max(1,d.max);liveProgress.value=d.value;
liveLabel.textContent=`${rows.find(r=>r.id===currentLayer)?.name??currentLayer??"Current layer"} · ${d.value}/${d.max}`;
status.textContent="Generating layers · previews appear here, inside the review panel.";
};
const preview=event=>{
const d=event.detail;if(!belongs(d)||!(d.blob instanceof Blob))return;
currentLayer=posterLayerId(d)??currentLayer;live.hidden=false;liveImage.hidden=false;
liveLabel.textContent=`${rows.find(r=>r.id===currentLayer)?.name??currentLayer??"Current layer"} · denoising preview`;
liveImage.src=replacePreviewUrl(previewState,d.blob);
};
const decoded=event=>{
const d=event.detail;if(!belongs(d))return;
for(const ready of d.output?.poster_layer_ready??[]){
currentLayer=ready.id;replacePreviewUrl(previewState,null);live.hidden=false;liveImage.hidden=false;liveProgress.hidden=true;
liveLabel.textContent=`${rows.find(r=>r.id===ready.id)?.name??ready.id} · decoded RGBA`;
liveImage.src=api.apiURL(`/view?${new URLSearchParams(ready.thumbnail)}`);
const thumbnail=layerImages.get(ready.id);if(thumbnail)thumbnail.src=liveImage.src;
}
};
const started=()=>{replacePreviewUrl(previewState,null);currentLayer=null;live.hidden=true;liveImage.removeAttribute("src");liveImage.hidden=true;};
const finished=()=>{if(!live.hidden){replacePreviewUrl(previewState,null);live.hidden=true;liveImage.removeAttribute("src");}};
const failed=event=>{if(belongs(event.detail)){liveProgress.hidden=true;liveLabel.textContent="Generation stopped · see execution details";}};
api.addEventListener("progress", progress);
const listeners={b_preview_with_metadata:preview,executed:decoded,execution_start:started,execution_success:finished,execution_error:failed,execution_interrupted:failed};
for(const [name,handler] of Object.entries(listeners))api.addEventListener(name,handler);
node.onRemoved = function () { disposed = true; replacePreviewUrl(previewState,null);api.removeEventListener("progress", progress);for(const [name,handler] of Object.entries(listeners))api.removeEventListener(name,handler); root.remove(); removed?.apply(this, arguments); };
queueMicrotask(() => { if (!disposed) load(node.properties.poster_review); });
node.setSize([760, 1100]);
};
},
});
+46
View File
@@ -0,0 +1,46 @@
// Small GPU inverse-warp preview. Textures belong to one editor and are disposed with it.
export function createDepthPreview() {
const canvas=document.createElement('canvas');
const gl=canvas.getContext('webgl',{alpha:true,premultipliedAlpha:true,antialias:false});
if(!gl)return null;
const compile=(type,source)=>{const shader=gl.createShader(type);gl.shaderSource(shader,source);gl.compileShader(shader);if(!gl.getShaderParameter(shader,gl.COMPILE_STATUS))throw Error(gl.getShaderInfoLog(shader));return shader;};
const vertex=compile(gl.VERTEX_SHADER,'attribute vec2 p; varying vec2 uv; void main(){uv=vec2(p.x*.5+.5,.5-p.y*.5);gl_Position=vec4(p,0.,1.);}');
const fragment=compile(gl.FRAGMENT_SHADER,`
precision highp float; varying vec2 uv; uniform sampler2D colorMap,depthMap;
uniform vec2 sizeScale,offset,travel; uniform float invDepth,push,strength,anchor,invert,opaque;
vec2 project(vec2 g,float d){return (g*(1.-push*d)+2.*travel*d-2.*offset)/sizeScale;}
void main(){vec2 g=uv*2.-1.;vec2 q=project(g,invDepth);
for(int i=0;i<2;i++){float d=texture2D(depthMap,clamp(q*.5+.5,0.,1.)).r;d=mix(d,1.-d,invert);q=project(g,invDepth*(1.+strength*(d-anchor)));}
vec2 t=q*.5+.5;vec4 c=texture2D(colorMap,clamp(t,0.,1.));
if(opaque<.5&&(t.x<0.||t.x>1.||t.y<0.||t.y>1.))c=vec4(0.);
gl_FragColor=c;
}`);
const program=gl.createProgram();gl.attachShader(program,vertex);gl.attachShader(program,fragment);gl.linkProgram(program);
if(!gl.getProgramParameter(program,gl.LINK_STATUS))throw Error(gl.getProgramInfoLog(program));
gl.deleteShader(vertex);gl.deleteShader(fragment);gl.useProgram(program);
const buffer=gl.createBuffer();gl.bindBuffer(gl.ARRAY_BUFFER,buffer);gl.bufferData(gl.ARRAY_BUFFER,new Float32Array([-1,-1,1,-1,-1,1,1,1]),gl.STATIC_DRAW);
const position=gl.getAttribLocation(program,'p');gl.enableVertexAttribArray(position);gl.vertexAttribPointer(position,2,gl.FLOAT,false,0,0);
const uniforms=Object.fromEntries(['colorMap','depthMap','sizeScale','offset','travel','invDepth','push','strength','anchor','invert','opaque'].map(k=>[k,gl.getUniformLocation(program,k)]));
const textures=new Map();
function upload(unit,data,width,height){const texture=gl.createTexture();gl.activeTexture(gl.TEXTURE0+unit);gl.bindTexture(gl.TEXTURE_2D,texture);gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL,true);
if(width)gl.texImage2D(gl.TEXTURE_2D,0,gl.RGBA,width,height,0,gl.RGBA,gl.UNSIGNED_BYTE,data);else gl.texImage2D(gl.TEXTURE_2D,0,gl.RGBA,gl.RGBA,gl.UNSIGNED_BYTE,data);
for(const key of [gl.TEXTURE_MIN_FILTER,gl.TEXTURE_MAG_FILTER])gl.texParameteri(gl.TEXTURE_2D,key,gl.LINEAR);
for(const key of [gl.TEXTURE_WRAP_S,gl.TEXTURE_WRAP_T])gl.texParameteri(gl.TEXTURE_2D,key,gl.CLAMP_TO_EDGE);return texture;}
return {
draw(ctx,p,s,value,width,height){
if(!p.relief||s.relief_scope==='off'||!s.relief_strength||(!p.background&&(s.relief_scope!=='background + artwork'||p.kind==='text')))return false;
if(canvas.width!==Math.round(width)||canvas.height!==Math.round(height)){canvas.width=Math.round(width);canvas.height=Math.round(height);}
gl.viewport(0,0,canvas.width,canvas.height);
let pair=textures.get(p);
if(!pair){const data=new Uint8Array(32*32*4);p.relief.forEach((d,i)=>{data[i*4]=data[i*4+1]=data[i*4+2]=Math.round(d*255);data[i*4+3]=255;});pair=[upload(0,p.bitmap),upload(1,data,32,32)];textures.set(p,pair);}
pair.forEach((texture,i)=>{gl.activeTexture(gl.TEXTURE0+i);gl.bindTexture(gl.TEXTURE_2D,texture);});gl.uniform1i(uniforms.colorMap,0);gl.uniform1i(uniforms.depthMap,1);
const depth=p.background?s.background_depth:p.depth,scale=p.background?s.overscan:p.scale;
const fit=!p.background&&s.layer_fit==='contain'?Math.min:Math.max,cover=fit(width/p.width,height/p.height);
gl.uniform2f(uniforms.sizeScale,p.width*cover/width*scale,p.height*cover/height*scale);gl.uniform2f(uniforms.offset,p.background?0:p.offset_x||0,p.background?0:p.offset_y||0);gl.uniform2f(uniforms.travel,value*s.travel_x,value*s.travel_y);
for(const [key,v] of Object.entries({invDepth:1/depth,push:value*s.push_in,strength:s.relief_strength,anchor:s.relief_anchor,invert:s.depth_invert?1:0,opaque:p.background?1:0}))gl.uniform1f(uniforms[key],v);
gl.drawArrays(gl.TRIANGLE_STRIP,0,4);ctx.drawImage(canvas,0,0,width,height);return true;
},
clear(){for(const pair of textures.values())for(const texture of pair)gl.deleteTexture(texture);textures.clear();},
dispose(){this.clear();gl.deleteBuffer(buffer);gl.deleteProgram(program);gl.getExtension('WEBGL_lose_context')?.loseContext();}
};
}
+96
View File
@@ -0,0 +1,96 @@
export const CAMERA_PRESETS = {
Gentle: {motion: "loop", travel_x: .06, travel_y: .015, push_in: .02},
Reveal: {motion: "glide", travel_x: .22, travel_y: .025, push_in: .04},
Punchy: {motion: "bursts", travel_x: .35, travel_y: .035, push_in: .06},
Still: {motion: "locked", travel_x: 0, travel_y: 0, push_in: 0},
};
export function cameraValue(t, motion) {
if (motion === "locked") return 0;
if (motion === "loop") return Math.sin(t * Math.PI * 2);
const smooth = u => u * u * (3 - 2 * u);
if (motion === "glide") return 2 * smooth(t) - 1;
const keys = [[0,-1],[.1,-1],[.28,.4],[.45,.4],[.61,-.3],[.73,-.3],[.92,1],[1,1]];
for (let i = 1; i < keys.length; i++) {
const [a, x] = keys[i-1], [b, y] = keys[i];
if (t <= b) return x + (y-x) * smooth((t-a)/(b-a));
}
return 1;
}
export function plateRect(p, s, phase, width, height) {
const depth = p.background ? s.background_depth : p.depth;
const value = cameraValue(phase, s.motion);
const zoom = depth / (depth - value * s.push_in);
const scale = p.background ? s.overscan : p.scale;
const fit = !p.background && s.layer_fit === "contain" ? Math.min : Math.max;
const cover = fit(width / p.width, height / p.height);
const w = p.width * cover * scale * zoom, h = p.height * cover * scale * zoom;
return {x: (width-w)/2 + ((p.offset_x || 0)-value*s.travel_x/depth)*zoom*width,
y: (height-h)/2 + ((p.offset_y || 0)-value*s.travel_y/depth)*zoom*height, w, h};
}
export function viewport(width, height, aspect) {
const w = Math.min(width, height * aspect), h = w / aspect;
return {x: (width-w)/2, y: (height-h)/2, w, h};
}
function borderImage(ctx, image, r, width, height) {
const iw = image.width, ih = image.height;
// Match the renderer's clamped background edges, not invented scenery.
if (r.x > 0) ctx.drawImage(image, 0, 0, 1, ih, 0, r.y, r.x, r.h);
if (r.y > 0) ctx.drawImage(image, 0, 0, iw, 1, r.x, 0, r.w, r.y);
if (r.x+r.w < width) ctx.drawImage(image, iw-1, 0, 1, ih, r.x+r.w, r.y, width-r.x-r.w, r.h);
if (r.y+r.h < height) ctx.drawImage(image, 0, ih-1, iw, 1, r.x, r.y+r.h, r.w, height-r.y-r.h);
for (const [sx, dx, dw] of [[0,0,r.x],[iw-1,r.x+r.w,width-r.x-r.w]])
for (const [sy, dy, dh] of [[0,0,r.y],[ih-1,r.y+r.h,height-r.y-r.h]])
if (dw > 0 && dh > 0) ctx.drawImage(image,sx,sy,1,1,dx,dy,dw,dh);
}
export function drawScene(canvas, plates, s, phase, depthView=false, relief=null) {
const ctx = canvas.getContext("2d"), w = canvas.width, h = canvas.height;
ctx.clearRect(0,0,w,h); ctx.fillStyle="#0b111c"; ctx.fillRect(0,0,w,h);
if (depthView) {
const ordered = [...plates].reverse();
ordered.forEach((p,i) => {
const depth = p.background ? s.background_depth : p.depth;
const x = 18 + depth / s.background_depth * (w-170), y = 30 + i * Math.min(32, (h-60)/ordered.length);
ctx.fillStyle = `hsl(${190+i*17} 40% 24%)`; ctx.fillRect(x,y,145,24);
ctx.fillStyle="#e4efff"; ctx.font="12px system-ui"; ctx.fillText(`${p.name.slice(0,15)} · ${depth}`,x+5,y+16);
});
ctx.fillStyle="#91a6be"; ctx.font="12px system-ui"; ctx.fillText("NEAR / FASTER",12,h-12); ctx.fillText("FAR / SLOWER",w-110,h-12);
return;
}
const v = viewport(w,h,s.width/s.height);
ctx.save(); ctx.translate(v.x,v.y); ctx.beginPath(); ctx.rect(0,0,v.w,v.h); ctx.clip();
for (const p of plates) {
if (!p.bitmap) continue;
const r = plateRect(p,s,phase,v.w,v.h);
ctx.globalAlpha = p.background ? 1 : p.opacity;
if(relief?.draw(ctx,p,s,cameraValue(phase,s.motion),v.w,v.h))continue;
if (p.background) borderImage(ctx,p.bitmap,r,v.w,v.h);
ctx.drawImage(p.bitmap,r.x,r.y,r.w,r.h);
}
ctx.restore(); ctx.strokeStyle="#597088"; ctx.strokeRect(v.x+.5,v.y+.5,v.w-1,v.h-1);
}
export function demoLayers(kind) {
const make = (name, depth, draw, background=false) => {
const bitmap = document.createElement("canvas"); bitmap.width=320; bitmap.height=200;
const context=bitmap.getContext("2d");context.scale(.5,.5);draw(context);
return {name, depth, bitmap, width:640, height:400, scale:1.04, offset_x:0, offset_y:0, opacity:1, background};
};
const background=make("Background",12,c=>{c.fillStyle=kind==="design"?"#efe8da":"#182c47";c.fillRect(0,0,640,400);},true);
if (kind === "design") return [background,
make("Color field",9,c=>{c.fillStyle="#df7154";c.beginPath();c.arc(360,225,160,0,7);c.fill();}),
make("Typography",5,c=>{c.fillStyle="#152a3c";c.font="bold 85px sans-serif";c.fillText("MOVE",45,145);c.fillText("IN DEPTH",45,235);}),
make("Accent",2,c=>{c.fillStyle="#e3bc36";c.fillRect(430,270,145,60);})];
if (kind === "product") return [background,
make("Backdrop",9,c=>{c.fillStyle="#426275";c.beginPath();c.ellipse(320,290,230,65,0,0,7);c.fill();}),
make("Object",4,c=>{c.fillStyle="#efa86b";c.fillRect(220,110,180,180);c.fillStyle="#b97349";c.beginPath();c.moveTo(400,110);c.lineTo(455,70);c.lineTo(455,245);c.lineTo(400,290);c.fill();c.fillStyle="#ffcf8e";c.beginPath();c.moveTo(220,110);c.lineTo(270,70);c.lineTo(455,70);c.lineTo(400,110);c.fill();}),
make("Foreground",2,c=>{c.fillStyle="#82c6bc";c.beginPath();c.arc(150,285,50,0,7);c.fill();})];
return [background,
make("Distant peaks",9,c=>{c.fillStyle="#607d94";c.beginPath();c.moveTo(0,300);c.lineTo(140,120);c.lineTo(265,280);c.lineTo(440,80);c.lineTo(640,310);c.lineTo(640,400);c.lineTo(0,400);c.fill();}),
make("Middle ground",5,c=>{c.fillStyle="#315d60";c.beginPath();c.moveTo(0,310);c.quadraticCurveTo(280,180,640,300);c.lineTo(640,400);c.lineTo(0,400);c.fill();}),
make("Near foliage",2,c=>{c.fillStyle="#91b18c";for(const [x,y] of [[20,335],[75,370],[570,350],[635,320]]){c.beginPath();c.ellipse(x,y,80,130,-.3,0,7);c.fill();}})];
}
+19
View File
@@ -0,0 +1,19 @@
export function ownsPosterEvent(nodeId, event) {
const id=String(nodeId);
return [event.node, event.node_id, event.nodeId, event.display_node, event.displayNodeId, event.parentNodeId, event.realNodeId]
.some(value=>value!=null&&(String(value)===id||String(value).startsWith(id+'.')));
}
export function posterLayerId(event) {
for(const value of [event.realNodeId,event.nodeId,event.node]){
const match=String(value??'').match(/(?:^|\.)((?:layer_)\d+)_(?:sample|decode|asset)$/);
if(match)return match[1];
}
return null;
}
export function replacePreviewUrl(state, blob, urls=URL) {
if(state.url)urls.revokeObjectURL(state.url);
state.url=blob?urls.createObjectURL(blob):null;
return state.url;
}
+72
View File
@@ -0,0 +1,72 @@
export function mappingRange(row, count) {
return [row.start_frame ?? 0, row.end_frame ?? count];
}
export function moveMappingRange(start, end, delta, count) {
const length = end-start;
start=Math.max(0,Math.min(count-length,start+delta));
return [start,start+length];
}
export function createAudioMappingPanel({settings,targets,onChange,onSeek}) {
const element=document.createElement("div");
const summary=document.createElement("div");summary.textContent="Envelope timeline · select a parameter badge to edit its range";
summary.style.cssText="cursor:pointer;margin:10px 0";
const timeline=document.createElement("canvas");timeline.width=640;timeline.height=150;
timeline.style.cssText="width:100%;height:150px;background:#0c111a;display:block;touch-action:none";
timeline.setAttribute("aria-label","Envelope timeline: drag selected range handles, or click to seek");
const readout=document.createElement("div");readout.style.cssText="margin:6px 0;color:#adbed5";
element.append(summary,timeline,readout);
let data=null,frame=0,selected=0,drag=null;
const read=()=>JSON.parse(settings.value);
const commit=value=>{settings.value=JSON.stringify(value);onChange();};
function draw() {
const ctx=timeline.getContext("2d"),w=timeline.width,h=timeline.height;
ctx.clearRect(0,0,w,h);
if(!data){ctx.fillStyle="#acbbd0";ctx.fillText("Run once to load envelopes and shot boundaries",12,28);return;}
const colors=["#70baff","#ed97ca","#f6cc78"];
for(let lane=0;lane<3;lane++){
ctx.strokeStyle=colors[lane];ctx.beginPath();
data.envelopes[lane].forEach((v,i)=>{const x=i/Math.max(1,data.frames-1)*w,y=38+lane*43-v*27;i?ctx.lineTo(x,y):ctx.moveTo(x,y);});ctx.stroke();
ctx.fillStyle=colors[lane];ctx.fillText(["Kick","Snare","Hat"][lane],5,12+lane*43);
}
for(const shot of (data.segments??[]).filter(s=>s.trigger==="authored_shot")){
const x=shot.start_frame/data.frames*w;ctx.strokeStyle="#52637b";ctx.beginPath();ctx.moveTo(x,0);ctx.lineTo(x,h);ctx.stroke();
ctx.fillStyle="#bbc8da";ctx.fillText(`Shot ${shot.shot}`,x+4,h-5);
}
const mapping=read().audio_mappings?.[selected];
if(mapping){
const [a,b]=mappingRange(mapping,data.frames),x=a/data.frames*w,right=b/data.frames*w;
ctx.fillStyle="#ffd67522";ctx.fillRect(x,0,right-x,h);ctx.strokeStyle="#ffd675";ctx.strokeRect(x,1,right-x,h-2);
ctx.fillStyle="#ffd675";ctx.fillRect(x,0,5,h);ctx.fillRect(right-5,0,5,h);
const value=data.curves?.[mapping.target]?.[frame];
readout.textContent=`${mapping.target}: ${value===undefined?"not rendered":value.toFixed(3)} · frame ${frame} · last render. Range [${a}, ${b})`;
}else readout.textContent=`Frame ${frame} · Add a mapping, then drag its range on the timeline.`;
ctx.strokeStyle="#ffffff";ctx.beginPath();ctx.moveTo(frame/data.frames*w,0);ctx.lineTo(frame/data.frames*w,h);ctx.stroke();
}
function rebuild() {
const mappings=read().audio_mappings??[];
selected=Math.min(selected,Math.max(0,mappings.length-1));draw();
}
const position=e=>Math.max(0,Math.min(data.frames,Math.round((e.clientX-timeline.getBoundingClientRect().left)/timeline.getBoundingClientRect().width*data.frames)));
timeline.onpointerdown=e=>{
if(!data)return;
const row=read().audio_mappings?.[selected],p=position(e);
if(row){const [a,b]=mappingRange(row,data.frames),tolerance=Math.max(1,data.frames*8/timeline.getBoundingClientRect().width);
const mode=Math.abs(p-a)<=tolerance?"start":Math.abs(p-b)<=tolerance?"end":p>a&&p<b?"move":null;
if(mode){drag={mode,start:a,end:b,anchor:p};timeline.setPointerCapture(e.pointerId);e.preventDefault();return;}}
onSeek(Math.min(data.frames-1,p));
};
timeline.onpointermove=e=>{
if(!drag)return;
const p=position(e),config=read(),row=config.audio_mappings[selected];
let a=drag.start,b=drag.end;
if(drag.mode==="start")a=Math.min(p,b-1);
else if(drag.mode==="end")b=Math.max(p,a+1);
else [a,b]=moveMappingRange(a,b,p-drag.anchor,data.frames);
if(config.audio_mappings.some((r,i)=>i!==selected&&r.enabled!==false&&row.enabled!==false&&r.target===row.target&&(r.start_frame??0)<b&&a<(r.end_frame??Infinity))){readout.textContent="Range overlaps an existing assignment; move was not applied.";return;}
row.start_frame=a;row.end_frame=b;commit(config);draw();
};
timeline.onpointerup=timeline.onpointercancel=()=>{if(drag){drag=null;rebuild();}};
return {element,rebuild,select(index){selected=index;draw();},frameCount(){return data?.frames??null;},update(preview,currentFrame){data=preview;frame=currentFrame;draw();}};
}
+159
View File
@@ -0,0 +1,159 @@
import { EFFECT_GROUPS } from "./scan_previs.js";
export function mappingTarget(key) {
return ({base_brightness:"brightness",base_saturation:"saturation",glow_intensity:"glow"})[key] ?? key;
}
export function normalizeCutRange(settings, changed=null) {
if(settings.min_cut_frames>settings.max_cut_frames){
if(changed==='min_cut_frames')settings.max_cut_frames=settings.min_cut_frames;
else if(changed==='max_cut_frames')settings.min_cut_frames=settings.max_cut_frames;
else [settings.min_cut_frames,settings.max_cut_frames]=[settings.max_cut_frames,settings.min_cut_frames];
}
return settings;
}
export function mappingConflict(rows, candidate, skip=-1) {
if(candidate.enabled===false)return false;
return rows.some((r,i)=>i!==skip&&r.enabled!==false&&r.target===candidate.target&&
(r.start_frame??0)<(candidate.end_frame??Infinity)&&(candidate.start_frame??0)<(r.end_frame??Infinity));
}
export function createScanControls({node,settings,options,onChange,mappingPanel,onSolo}) {
const element=document.createElement("div"),sources=document.createElement("div"),tabs=document.createElement("div"),body=document.createElement("div"),notice=document.createElement("div");
element.className="fl-scan-controls";
const style=document.createElement("style");style.textContent=`
.fl-scan-controls button{background:#26364a;color:#e7f1ff;border:1px solid #58708f;border-radius:4px;padding:3px 5px;font:11px system-ui;cursor:pointer;min-height:24px}
.fl-scan-controls button[aria-selected=true],.fl-scan-controls button[aria-pressed=true]{background:#326397;border-color:#91ceff}
.fl-scan-controls button:disabled{opacity:.4;cursor:default}
.fl-scan-controls input{background:#101b2a;color:white;border:1px solid #58708f;border-radius:4px;min-width:0;padding:4px;box-sizing:border-box}
.fl-scan-controls input[type=range]{padding:0;accent-color:#7bbfff;width:100%}
.fl-scan-controls input:invalid{border-color:#ff8989}
.fl-scan-controls .scan-body{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:5px;align-items:start}
.fl-scan-controls .scan-toolbar{grid-column:1/-1;display:flex;flex-wrap:wrap;gap:4px;margin-bottom:3px}
.fl-scan-controls .scan-card{border:1px solid #35465c;border-radius:5px;padding:6px;min-width:0}
.fl-scan-controls .scan-card:has(.scan-mapping){grid-column:1/-1}
.fl-scan-controls .scan-card strong{font-size:11px;font-weight:500;line-height:1.2}
.fl-scan-controls .scan-card>label:empty{display:none}
.fl-scan-controls.assigning .scan-card[data-target]{border-color:#91ceff;background:#193149}
.fl-scan-controls .scan-card.drop{outline:2px solid #c7e5ff}
.fl-scan-controls .scan-mapping{padding:8px;border-left:3px solid #80bdff;background:#111e2d;margin-top:8px}
.fl-scan-controls label{font-size:11px;color:#bfd0e5}
`;
sources.style.cssText=tabs.style.cssText="display:flex;gap:4px;flex-wrap:wrap;margin-bottom:6px";
body.className="scan-body";
tabs.setAttribute("role","tablist");
notice.style.cssText="font-size:11px;color:#ffd08a;line-height:1.5";notice.setAttribute("role","status");
element.append(style,sources,tabs,notice,body);
const read=()=>JSON.parse(settings.value),targets=options.scan_mapping_targets;
const colors=["#70baff","#ed97ca","#f6cc78"],names=["Kick","Snare","Hat"];
let group=0,armed=null,expanded=-1;
const btn=(text,parent,action)=>{const b=document.createElement("button");b.textContent=text;b.type="button";b.onclick=action;parent.append(b);return b;};
function change(){node.graph?.setDirtyCanvas(true,true);onChange();}
function commit(config){settings.value=JSON.stringify(config);change();mappingPanel.rebuild();}
function arm(source){armed=source;element.classList.toggle("assigning",source!==null);[...sources.children].forEach((b,i)=>b.setAttribute("aria-pressed",String(i===source)));notice.textContent=source===null?"":`Click a highlighted parameter to assign ${names[source]}. Escape cancels.`;}
names.forEach((name,i)=>{
const b=btn(`${i+1} · ${name}`,sources,()=>arm(armed===i?null:i));b.draggable=true;b.style.borderColor=colors[i];
b.title="Drag onto a highlighted parameter, or click then select a parameter.";
b.ondragstart=e=>{e.dataTransfer.setData("application/x-fl-envelope",String(i));e.dataTransfer.effectAllowed="copy";arm(i);};
b.ondragend=()=>arm(null);
});
element.onkeydown=e=>{if(e.key==="Escape")arm(null);};
const entries=Object.entries(EFFECT_GROUPS),titles=["Voxels","Camera","Reveals","Overlay","Finish","Layers"];
entries.forEach((entry,i)=>{const b=btn(titles[i],tabs,()=>{group=i;rebuild();});b.setAttribute("role","tab");b.onkeydown=e=>{if(!['ArrowLeft','ArrowRight','Home','End'].includes(e.key))return;e.preventDefault();group=e.key==='Home'?0:e.key==='End'?entries.length-1:(i+(e.key==='ArrowRight'?1:entries.length-1))%entries.length;rebuild();tabs.children[group].focus();};});
function meta(key){
const spec=options.widget_specs?.[key]??{};
return {default:spec.default,range:spec.min===undefined?null:[spec.min,spec.max],help:spec.tooltip??`${key.replaceAll("_"," ")}.`,choices:options.scan_choices[key],...options.scan_controls?.[key]};
}
const widget=key=>node.widgets.find(w=>w.name===key);
for(const key of Object.values(EFFECT_GROUPS).flat()){
const w=widget(key);if(w){w.hidden=true;w.type="converted-widget";w.computeSize=()=>[0,0];w.computedHeight=0;if(w.element)w.element.style.display="none";}
}
const value=key=>widget(key)?.value??read()[key];
const linked=key=>node.inputs?.some(i=>i.name===key&&i.link!=null);
function setValue(key,v){
const w=widget(key);
if(w){w.value=v;w.callback?.(v);}
else{
const c=read();c[key]=v;
if(key==='min_cut_frames'||key==='max_cut_frames'){
normalizeCutRange(c,key);
for(const name of ['min_cut_frames','max_cut_frames']){
for(const input of element.querySelectorAll(`[aria-label="${name.replaceAll('_',' ')}"], [aria-label="${name} slider"]`))input.value=c[name];
}
}
settings.value=JSON.stringify(c);
}
change();
}
function newMapping(target,source,start=0,end=null){const bounds=targets[target];const key=Object.values(EFFECT_GROUPS).flat().find(k=>mappingTarget(k)===target);const base=Math.max(bounds[0],Math.min(bounds[1],Number(value(key))));return {enabled:true,source,target,minimum:base,maximum:Math.min(bounds[1],base+(bounds[1]-bounds[0])*.1),start_frame:start,end_frame:end,invert:false,smoothing:0};}
function assign(target,source,host){
arm(null);const c=read();c.audio_mappings??=[];
const matches=c.audio_mappings.map((r,i)=>r.target===target?i:-1).filter(i=>i>=0);
const add=(replace)=>{
if(replace){const i=matches.includes(expanded)?expanded:matches[0];c.audio_mappings[i].source=source;expanded=i;}
else{
let start=0,end=null;const ranges=c.audio_mappings.filter(r=>r.target===target&&r.enabled!==false).sort((a,b)=>(a.start_frame??0)-(b.start_frame??0));
for(const r of ranges){if((r.start_frame??0)>start){end=r.start_frame;break;}start=Math.max(start,r.end_frame??Infinity);}
if(!Number.isFinite(start)){notice.textContent="No unused time range. Shorten an existing range before adding another.";return;}
const count=mappingPanel.frameCount();if(count&&start>=count){notice.textContent="No unused frames remain. Shorten an existing range first.";return;}
c.audio_mappings.push(newMapping(target,source,start,end));expanded=c.audio_mappings.length-1;
}
commit(c);mappingPanel.select(expanded);rebuild();
};
if(matches.length){const choice=document.createElement("div");choice.textContent="Existing assignment: ";btn("Replace selected",choice,()=>add(true));btn("Add time range",choice,()=>add(false));btn("Cancel",choice,()=>choice.remove());host.append(choice);}
else add(false);
}
function mappingEditor(row,index,parent){
const box=document.createElement("div");box.className="scan-mapping";box.style.borderColor=colors[row.source];parent.append(box);
const grid=document.createElement("div");grid.style.cssText="display:grid;grid-template-columns:1fr 1fr;gap:7px";box.append(grid);
for(const [key,label] of [["minimum","Minimum"],["maximum","Maximum"],["start_frame","Start frame"],["end_frame","End frame (exclusive)"],["smoothing","Smoothing (seconds)"],["enabled","Enabled"],["invert","Invert"]]){
const wrap=document.createElement("label"),input=document.createElement("input");wrap.textContent=label;
const check=["enabled","invert"].includes(key);input.type=check?"checkbox":"number";
input.style.width=check?"auto":"100%";input.setAttribute("aria-label",`${row.target} mapping ${index+1} ${label}`);
if(check)input.checked=row[key]??(key==="enabled");else{input.value=row[key]??"";input.placeholder=key==="end_frame"?"Whole clip":"";input.step=key.includes("frame")?1:.01;
const range=key==="minimum"||key==="maximum"?targets[row.target]:[0,key==="smoothing"?2:2147483647];[input.min,input.max]=range;}
input.onchange=()=>{if(!input.checkValidity())return;const c=read(),next={...c.audio_mappings[index],[key]:check?input.checked:input.value===""&&key==="end_frame"?null:Number(input.value)};
const count=mappingPanel.frameCount();
if((next.end_frame??Infinity)<=next.start_frame||(count&&(next.start_frame>=count||(next.end_frame??count)>count))||mappingConflict(c.audio_mappings,next,index)){notice.textContent="Invalid or overlapping time range. Change was not applied.";input.value=row[key]??"";if(check)input.checked=row[key];return;}
c.audio_mappings[index]=next;commit(c);notice.textContent="";rebuild();};
wrap.append(input);grid.append(wrap);
}
btn("Reset mapping",box,()=>{const c=read(),old=c.audio_mappings[index];c.audio_mappings[index]={...newMapping(row.target,row.source,old.start_frame,old.end_frame),enabled:old.enabled};commit(c);rebuild();});
btn("Remove mapping",box,()=>{const c=read();c.audio_mappings.splice(index,1);expanded=-1;commit(c);rebuild();});
btn("Edit time range",box,()=>{mappingPanel.select(index);mappingPanel.element.scrollIntoView({block:"nearest"});});
}
function rebuild(){
settings.value=JSON.stringify(normalizeCutRange(read()));
body.replaceChildren();[...tabs.children].forEach((b,i)=>{b.setAttribute("aria-selected",String(i===group));b.tabIndex=i===group?0:-1;});
const toolbar=document.createElement("div");toolbar.className="scan-toolbar";body.append(toolbar);
btn("Solo",toolbar,()=>onSolo(["voxel","depth","cursors","edges","finish","layers"][group]));
btn("All effects",toolbar,()=>onSolo("all"));
btn("Reset tab values",toolbar,()=>{const confirm=document.createElement("div");confirm.textContent="Restore factory values? Mappings stay unchanged. ";btn("Confirm reset",confirm,()=>{for(const k of entries[group][1])if(!linked(k))setValue(k,meta(k).default);rebuild();});btn("Cancel",confirm,()=>confirm.remove());toolbar.append(confirm);});
for(const key of entries[group][1]){
const w=widget(key);if(w){w.hidden=true;w.type="converted-widget";w.computeSize=()=>[0,0];w.computedHeight=0;if(w.element)w.element.style.display="none";}
const m=meta(key),v=value(key),target=mappingTarget(key),eligible=!!targets[target]&&!linked(key);
const card=document.createElement("div");card.className="scan-card";if(eligible)card.dataset.target=target;body.append(card);
if(eligible){card.tabIndex=0;card.setAttribute("aria-label",`Assign envelope to ${key.replaceAll('_',' ')}`);card.onkeydown=e=>{if(armed!==null&&e.target===card&&['Enter',' '].includes(e.key)){e.preventDefault();assign(target,armed,card);}};}
const head=document.createElement("div");head.style.cssText="display:flex;align-items:center;gap:4px;margin-bottom:4px";card.append(head);
const label=document.createElement("strong");label.textContent=key.replaceAll("_"," ");label.style.flex=1;head.append(label);
const help=btn("?",head,()=>{let text=card.querySelector('.scan-help');if(text){text.remove();return;}text=document.createElement('p');text.className='scan-help';text.textContent=m.help;card.append(text);});help.title=m.help;help.setAttribute("aria-label",`Help: ${key}`);
const reset=btn("↺",head,()=>{setValue(key,m.default);rebuild();});reset.setAttribute("aria-label",`Reset ${key}`);reset.disabled=!!linked(key);reset.title="Reset base value; keep envelope assignments.";
if(typeof v==="number"){
const line=document.createElement("div");line.style.cssText="display:flex;gap:8px;align-items:center";card.append(line);
const range=m.range??targets[target];const step=["seed","surface_seed","cube_size","cursor_count","min_cut_frames","max_cut_frames","stack_count"].includes(key)?1:.01;
const number=document.createElement("input");number.type="number";number.value=v;number.step=step;number.style.cssText="width:64px;flex-shrink:0;font-size:11px";number.setAttribute("aria-label",key.replaceAll("_"," "));number.disabled=!!linked(key);
let slider;if(range){[number.min,number.max]=range;number.title=`Range: ${range[0]} to ${range[1]}`;if(!key.includes("seed")){slider=document.createElement("input");slider.type="range";slider.min=range[0];slider.max=range[1];slider.step=step;slider.value=v;slider.disabled=number.disabled;slider.title=number.title;slider.setAttribute("aria-label",`${key} slider`);line.append(slider);}}
number.oninput=()=>{if(number.value===""||!number.checkValidity())return;setValue(key,Number(number.value));if(slider)slider.value=number.value;};
if(slider)slider.oninput=()=>{number.value=slider.value;setValue(key,Number(slider.value));};line.append(number);
}else{const line=document.createElement("div");line.style.cssText="display:flex;gap:4px;flex-wrap:wrap";card.append(line);for(const choice of m.choices??[]){const b=btn(choice.replaceAll("_"," "),line,()=>{setValue(key,choice);rebuild();});b.setAttribute("aria-pressed",String(v===choice));b.disabled=!!linked(key);}}
if(linked(key)){const text=document.createElement("p");text.textContent="Controlled by connected input.";card.append(text);}
card.addEventListener("click",e=>{if(armed!==null&&eligible){e.preventDefault();e.stopPropagation();assign(target,armed,card);}},true);
card.ondragover=e=>{if(eligible&&e.dataTransfer.types.includes("application/x-fl-envelope")){e.preventDefault();card.classList.add("drop");}};
card.ondragleave=()=>card.classList.remove("drop");card.ondrop=e=>{e.preventDefault();card.classList.remove("drop");const raw=e.dataTransfer.getData("application/x-fl-envelope");if(eligible&&["0","1","2"].includes(raw))assign(target,Number(raw),card);};
read().audio_mappings?.forEach((row,index)=>{if(row.target!==target)return;const badge=btn(`${names[row.source]} · ${row.enabled===false?"off":"mapped"} · ${row.start_frame??0}–${row.end_frame??"end"}`,card,()=>{expanded=expanded===index?-1:index;mappingPanel.select(index);rebuild();});badge.style.borderColor=colors[row.source];if(expanded===index)mappingEditor(row,index,card);});
if(eligible&&read().audio_mappings?.some(r=>r.target===target)){const meter=document.createElement("meter"),text=document.createElement("label");meter.hidden=true;meter.min=targets[target][0];meter.max=targets[target][1];meter.dataset.renderTarget=target;meter.style.width="100%";meter.setAttribute("aria-label",`${target} last rendered value`);text.dataset.renderLabel=target;card.append(meter,text);}
}
}
return {element,rebuild,update(preview,frame){element.querySelectorAll('[data-render-target]').forEach(m=>{const v=preview.curves?.[m.dataset.renderTarget]?.[frame];m.hidden=v===undefined;if(v!==undefined)m.value=v;const label=element.querySelector(`[data-render-label="${m.dataset.renderTarget}"]`);label.textContent=v===undefined?"Not rendered":`Last render: ${v.toFixed(3)} · frame ${frame}`;});}};
}
+136
View File
@@ -0,0 +1,136 @@
export const EFFECT_GROUPS = {
"Voxel surface": ["cube_size", "relief", "animation", "speed", "surface_seed"],
"Depth camera": ["motion_mode", "parallax_scope", "orbit_degrees", "depth_relief", "scene_scale", "parallax_strength", "offset_x", "offset_y", "dolly", "steady_depth"],
"Cursor reveals": ["cursor_count", "cursor_scale", "cursor_activity", "reveal_size", "reveal_strength", "voxel_weight", "edge_weight", "depth_weight", "depth_style", "seed"],
"Digital overlay": ["normal_mix", "hud_opacity", "pose_opacity", "motion_strength", "min_cut_frames", "max_cut_frames"],
"Color & glow": ["base_brightness", "brightness_intensity", "base_saturation", "saturation_intensity", "edge_threshold", "glow_intensity", "envelope_intensity", "glow_color", "blend_mode"],
"Layer stack": ["stack_count", "stack_spacing", "stack_x", "stack_y", "stack_rotation", "stack_opacity", "stack_palette", "window_order", "window_blend", "voxel_opacity", "edge_opacity", "depth_opacity", "window_fade_in", "window_fade_out"],
};
// Small synthetic surface, not model output. Coordinates are normalized to the canvas.
export function demoPoint(x, y, depth, s, time) {
const native=s.motion_mode==="depth_parallax", strength=native?s.parallax_strength:0;
const z=1.5+(1-depth)*s.depth_relief, pivot=1.5+(1-s.steady_depth)*s.depth_relief;
const angle=Math.sin(time*.7)*s.orbit_degrees*Math.PI/180;
const px=(x-.5)*Math.cos(angle)+(z-pivot)*Math.sin(angle)*.3;
const zoom=(pivot-strength*s.dolly)/pivot*z/Math.max(.1,z-strength*s.dolly);
return [.5+(px+strength*s.offset_x*(1/z-1/pivot))*s.scene_scale*zoom,
.5+((y-.5)+strength*s.offset_y*(1/z-1/pivot))*s.scene_scale*zoom];
}
export function drawPrevis(canvas, s, time, view, buffer = canvas.ownerDocument.createElement("canvas")) {
let ctx=canvas.getContext("2d");
const w=canvas.width, h=canvas.height;
if(buffer.width!==w)buffer.width=w;
if(buffer.height!==h)buffer.height=h;
ctx.clearRect(0,0,w,h);ctx.fillStyle="#080f1c";ctx.fillRect(0,0,w,h);
const grid=Math.max(8,Math.round(s.cube_size/640*w)), cells=Math.ceil(w/grid);
const pulse=Math.pow(Math.max(0,Math.cos(time*Math.PI*4)),8);
const surface=(kind,clip=false)=>{
const destination=ctx;
ctx=buffer.getContext("2d");ctx.clearRect(0,0,w,h);
for(let row=0;row<cells;row++)for(let col=0;col<cells;col++){
const x=(col+.5)/cells,y=(row+.5)/cells;
const radius=Math.hypot((x-.5)*1.4,(y-.5)*1.05),depth=Math.max(0,1-radius*2.3);
const cameraSettings=s.parallax_scope==="reveals_only"&&!clip&&kind!=="depth"?{...s,motion_mode:"current"}:s;
const [px,py]=demoPoint(x,y,depth,cameraSettings,time).map((v,i)=>v*(i?h:w));
const voxel=kind==="voxel"||(clip&&kind==="all");
const lift=voxel?depth*s.relief*18+Math.sin(time*s.speed*3+x*9+y*8+s.surface_seed)*s.animation*12:0;
if(kind==="depth")ctx.fillStyle=s.depth_style==="grayscale"?`rgb(${depth*255} ${depth*255} ${depth*255})`:
s.depth_style==="contours"?`hsl(190 80% ${Math.floor(depth*12)%2?60:15}%)`:`hsl(${(1-depth)*250} 80% 55%)`;
else ctx.fillStyle=voxel?`hsl(${210+x*110-y*80} 65% ${30+depth*35}%)`:`hsl(${215-depth*45+s.normal_mix*x*120} ${45+s.normal_mix*35}% ${15+depth*65}%)`;
if(voxel){ctx.fillStyle="#172940";ctx.fillRect(px-grid/2,py-lift,grid*.93,grid*.85+lift);ctx.fillStyle=`hsl(${210+x*110-y*80} 65% ${30+depth*35}%)`;}
ctx.fillRect(px-grid/2,py-lift,grid*(voxel?.86:1.04),grid*(voxel?.72:1.04));
}
ctx=destination;ctx.save();
if(kind==="all"||kind==="finish")ctx.filter=`brightness(${Math.max(0,s.base_brightness+s.brightness_intensity*pulse)}) saturate(${Math.max(0,s.base_saturation+s.saturation_intensity*pulse)})`;
ctx.drawImage(buffer,0,0);ctx.restore();
};
ctx.save();
if(view==="all"){ctx.translate(Math.sin(time*4)*s.motion_strength*4,Math.cos(time*3)*s.motion_strength*3);}
if(view==="all"||view==="layers"){
const step=Math.max(2,Math.round(w*(.009+.004*s.depth_relief)))*(s.stack_spacing??1);
const color=({cobalt:"#0905e6",cyan:"#05b3d9",magenta:"#cc0599",mono:"#a6a6b3"})[s.stack_palette??"cobalt"];
for(let level=s.stack_count??4;level>0;level--){
ctx.save();ctx.globalAlpha=s.stack_opacity??1;
ctx.translate(w/2+level*step*(s.stack_x??1),h/2+level*step*(s.stack_y??1)*(.65+.2*Math.sin(time)));
ctx.rotate(-level*(s.stack_rotation??0)*Math.PI/180);
ctx.fillStyle=level%2?"#0a0a0e":color;ctx.strokeStyle="#d9e6ff";
ctx.fillRect(-w*s.scene_scale/2,-h*s.scene_scale/2,w*s.scene_scale,h*s.scene_scale);
ctx.strokeRect(-w*s.scene_scale/2,-h*s.scene_scale/2,w*s.scene_scale,h*s.scene_scale);ctx.restore();
}
}
surface(view);
if(view==="all"||view==="cursors"||view==="layers"){
const count=Math.min(3,s.cursor_count),period=Math.max(.4,(s.min_cut_frames+s.max_cut_frames)/48);
const windows=[];
for(let i=0;i<count;i++){
const phase=(time/period+i*.31)%1;if(phase>s.cursor_activity)continue;
const x=(.12+i*.22+Math.sin(s.seed+i)*.06)*w,y=(.18+i*.17)*h;
const rw=w*.28*s.reveal_size*Math.min(1,phase*3),rh=h*.32*s.reveal_size*Math.min(1,phase*3);
const total=s.voxel_weight+s.edge_weight+s.depth_weight;
const pick=((Math.sin(s.seed+i*17+Math.floor(time/period))*43758.5453)%1+1)%1*total;
const kind=pick<s.voxel_weight?"voxel":pick<s.voxel_weight+s.edge_weight?"edges":"depth";
windows.push({i,phase,x,y,rw,rh,kind});
}
const priority=({voxel_on_top:"voxel",edge_on_top:"edges",depth_on_top:"depth"})[s.window_order];
if(priority)windows.sort((a,b)=>Number(a.kind===priority)-Number(b.kind===priority));
if(s.window_order==="random_on_snare")windows.sort((a,b)=>Math.sin(s.seed+a.i*79+Math.floor(time*2)*31)-Math.sin(s.seed+b.i*79+Math.floor(time*2)*31));
for(const {phase,x,y,rw,rh,kind} of windows){
const opacity=s[({voxel:"voxel_opacity",edges:"edge_opacity",depth:"depth_opacity"})[kind]]??1;
const fadeIn=s.window_fade_in?Math.min(1,Math.max(0,phase-.12)*period/s.window_fade_in):1;
const fadeOut=s.window_fade_out?Math.min(1,(1-phase)*period/s.window_fade_out):1;
ctx.save();ctx.globalAlpha=s.reveal_strength*opacity*fadeIn*fadeOut;
ctx.globalCompositeOperation=({normal:"source-over",screen:"screen",add:"lighter"})[s.window_blend??"normal"];
ctx.beginPath();ctx.rect(x,y,rw,rh);ctx.clip();
if(kind==="edges"){ctx.strokeStyle="#48f2ff";for(let j=0;j<10;j++)ctx.strokeRect(w*(.2+j*.03),h*(.15+j*.03),w*.3,h*.5);}
else surface(kind,true);
ctx.restore();ctx.strokeStyle=kind==="depth"?"#ffd36b":"#78f4ed";ctx.strokeRect(x,y,rw,rh);
ctx.save();ctx.translate(x+rw,y+rh);ctx.scale(s.cursor_scale,s.cursor_scale);ctx.fillStyle="#fff";ctx.strokeStyle="#0c1726";
ctx.beginPath();ctx.moveTo(0,0);ctx.lineTo(2,19);ctx.lineTo(7,13);ctx.lineTo(13,20);ctx.lineTo(17,17);ctx.lineTo(11,10);ctx.lineTo(19,8);ctx.closePath();ctx.fill();ctx.stroke();ctx.restore();
}
}
if(view==="all"||view==="edges"){
ctx.save();ctx.globalAlpha=s.hud_opacity;ctx.strokeStyle="#9ef66e";ctx.strokeRect(w*.3,h*.18,w*.4,h*.62);
ctx.fillStyle="#9ef66e";ctx.font="12px system-ui";ctx.fillText("SYNTHETIC DEPTH SUBJECT",w*.3,h*.16);
ctx.globalAlpha=s.pose_opacity;ctx.beginPath();ctx.moveTo(w*.5,h*.3);ctx.lineTo(w*.5,h*.58);ctx.lineTo(w*.38,h*.77);ctx.moveTo(w*.5,h*.58);ctx.lineTo(w*.62,h*.77);ctx.stroke();ctx.restore();
}
if(view==="all"||view==="finish"){
ctx.save();ctx.globalAlpha=Math.min(.8,(s.glow_intensity+s.envelope_intensity*pulse)*.35)*(1-s.edge_threshold);
ctx.globalCompositeOperation=s.blend_mode==="add"?"lighter":s.blend_mode;
ctx.shadowBlur=20;ctx.shadowColor=s.glow_color==="original"?"#75d8ff":s.glow_color;
ctx.strokeStyle=ctx.shadowColor;ctx.lineWidth=3;ctx.strokeRect(w*.28,h*.2,w*.44,h*.6);ctx.restore();
}
ctx.restore();
}
export function createPrevis(readSettings) {
const element=document.createElement("div"), tabs=document.createElement("div"), canvas=document.createElement("canvas"), note=document.createElement("p");
element.className="fl-scan-previs";canvas.width=320;canvas.height=320;
const buffer=document.createElement("canvas");
canvas.style.cssText="width:100%;display:block;background:#080f1c";
tabs.style.cssText="display:flex;gap:4px;flex-wrap:wrap;margin-bottom:8px";
let view="all",time=1,playing=true,active=true,visible=false,raf=null,last=0;
const buttons=[];
const redraw=()=>{drawPrevis(canvas,readSettings(),time,view,buffer);buttons.forEach(([button,key])=>button.style.background=key===view?"#345d91":"#293345");};
for(const [key,label] of [["all","All"],["voxel","Voxels"],["depth","Depth"],["cursors","Reveals"],["edges","Overlay"],["finish","Finish"],["layers","Layers"]]){
const b=document.createElement("button");b.textContent=label;b.style.cssText="color:white;border:1px solid #46536c;border-radius:4px;padding:5px";
b.onclick=()=>{view=key;redraw();};buttons.push([b,key]);tabs.append(b);
}
const pause=document.createElement("button");pause.textContent="Pause demo";pause.onclick=()=>{playing=!playing;pause.textContent=playing?"Pause demo":"Play demo";schedule();};
note.textContent="LIVE DEMO · Synthetic scene, approximate effects. No model or queue required. Audio pulses are illustrative; mappings, tracking, cut timing and occlusion must be checked in the render.";
note.style.cssText="color:#bdcce1;font-size:11px;line-height:1.5";
element.append(tabs,canvas,pause,note);
function tick(now){
raf=null;
if(now-last>=1000/12){time+=last?Math.min(.15,(now-last)/1000):0;last=now;redraw();}
schedule();
}
function schedule(){
if(active&&visible&&playing&&!document.hidden){if(raf===null)raf=requestAnimationFrame(tick);}
else {if(raf!==null)cancelAnimationFrame(raf);raf=null;last=0;}
}
const observer=new IntersectionObserver(entries=>{visible=entries[0].isIntersecting;schedule();});observer.observe(element);
document.addEventListener("visibilitychange",schedule);
return {element,redraw,select(key){view=key;redraw();},setActive(value){active=value;element.hidden=!value;schedule();},dispose(){active=false;schedule();observer.disconnect();document.removeEventListener("visibilitychange",schedule);}};
}