diff --git a/examples/basic_example.gif b/examples/basic_example.gif new file mode 100644 index 0000000..14adb81 Binary files /dev/null and b/examples/basic_example.gif differ diff --git a/examples/basic_example.json b/examples/basic_example.json index f34fe21..d3a87bc 100644 --- a/examples/basic_example.json +++ b/examples/basic_example.json @@ -870,31 +870,6 @@ "Node name for S&R": "NormalizedAmplitudeToGraph" } }, - { - "id": 379, - "type": "PreviewImage", - "pos": [ - -4188.528412070098, - -168.73949147146516 - ], - "size": { - "0": 1080, - "1": 290 - }, - "flags": {}, - "order": 31, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 626 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, { "id": 376, "type": "NormalizeAmplitude", @@ -975,46 +950,12 @@ 70 ] }, - { - "id": 316, - "type": "LoadAudio", - "pos": [ - -6400, - -600 - ], - "size": { - "0": 315, - "1": 82 - }, - "flags": {}, - "order": 6, - "mode": 0, - "outputs": [ - { - "name": "AUDIO", - "type": "AUDIO", - "links": [ - 541, - 656 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "LoadAudio" - }, - "widgets_values": [ - "Real Good Snippet.wav", - "Audio" - ] - }, { "id": 404, "type": "AudioToAmplitudeGraph", "pos": [ -5310, - -660 + -700 ], "size": { "0": 315, @@ -1050,59 +991,6 @@ 4000 ] }, - { - "id": 274, - "type": "ADE_AnimateDiffLoaderWithContext", - "pos": [ - -952.399951171875, - -535.7999633789063 - ], - "size": { - "0": 315, - "1": 122 - }, - "flags": { - "collapsed": false - }, - "order": 40, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 472 - }, - { - "name": "context_options", - "type": "CONTEXT_OPTIONS", - "link": 443, - "slot_index": 1 - }, - { - "name": "motion_lora", - "type": "MOTION_LORA", - "link": null - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 473 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffLoaderWithContext" - }, - "widgets_values": [ - "mm_sd_v15_v2.ckpt", - "sqrt_linear (AnimateDiff)" - ] - }, { "id": 268, "type": "BatchPromptSchedule", @@ -1205,7 +1093,7 @@ "Node name for S&R": "BatchPromptSchedule" }, "widgets_values": [ - "\"0\" :\"smile, (open mouth, laughing:`pw_a`)\",\n\"128\" :\"(open mouth, laughing:`pw_a`)\"", + "\"0\" :\"(laugh:`pw_a`)\",\n\"128\" :\"(laugh:`pw_a`)\"", 120, "RAW photo of korean woman, ", "highly detailed, realistic, portrait, simple background, turtleneck sweater", @@ -1217,77 +1105,6 @@ "color": "#232", "bgcolor": "#353" }, - { - "id": 271, - "type": "VHS_VideoCombine", - "pos": [ - -510, - 100 - ], - "size": { - "0": 800, - "1": 980 - }, - "flags": {}, - "order": 44, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 436 - }, - { - "name": "filename_prefix", - "type": "STRING", - "link": 695, - "widget": { - "name": "filename_prefix", - "config": [ - "STRING", - { - "default": "AnimateDiff" - } - ] - } - }, - { - "name": "frame_rate", - "type": "INT", - "link": 699, - "widget": { - "name": "frame_rate", - "config": [ - "INT", - { - "default": 8, - "min": 1, - "step": 1 - } - ] - } - } - ], - "outputs": [ - { - "name": "GIF", - "type": "GIF", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": [ - 12, - 0, - "AnimateDiff/AD", - "video/h264-mp4", - false, - true - ] - }, { "id": 302, "type": "AudioToFFTs", @@ -1428,147 +1245,6 @@ false ] }, - { - "id": 275, - "type": "ImpactKSamplerBasicPipe", - "pos": [ - -311.66975156067997, - -335.3463791916187 - ], - "size": { - "0": 315, - "1": 242 - }, - "flags": {}, - "order": 42, - "mode": 0, - "inputs": [ - { - "name": "basic_pipe", - "type": "BASIC_PIPE", - "link": 454 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 446 - } - ], - "outputs": [ - { - "name": "BASIC_PIPE", - "type": "BASIC_PIPE", - "links": null, - "shape": 3 - }, - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 448 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 447 - ], - "shape": 3, - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "ImpactKSamplerBasicPipe" - }, - "widgets_values": [ - 153435234221149, - "randomize", - 25, - 7, - "ddim", - "ddim_uniform", - 1 - ] - }, - { - "id": 405, - "type": "PreviewImage", - "pos": [ - -4930, - -660 - ], - "size": { - "0": 1950, - "1": 330 - }, - "flags": {}, - "order": 19, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 657 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 4, - "type": "CheckpointLoaderSimple", - "pos": [ - -2887.103581502704, - -429.99626155612657 - ], - "size": { - "0": 315, - "1": 98 - }, - "flags": { - "collapsed": false - }, - "order": 9, - "mode": 0, - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 471 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 427, - 458, - 707 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 382 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "juggernaut_final.safetensors" - ] - }, { "id": 436, "type": "CLIPTextEncode", @@ -1914,7 +1590,7 @@ "Node name for S&R": "Text Multiline" }, "widgets_values": [ - "AnimateDiff/[time(%Y-%m-%d)]/Music" + "AnimateDiff/[time(%Y-%m-%d)]/BasicExample" ] }, { @@ -1966,6 +1642,330 @@ "widgets_values": [ "" ] + }, + { + "id": 274, + "type": "ADE_AnimateDiffLoaderWithContext", + "pos": [ + -952.399951171875, + -535.7999633789063 + ], + "size": { + "0": 315, + "1": 122 + }, + "flags": { + "collapsed": false + }, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 472 + }, + { + "name": "context_options", + "type": "CONTEXT_OPTIONS", + "link": 443, + "slot_index": 1 + }, + { + "name": "motion_lora", + "type": "MOTION_LORA", + "link": null + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 473 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ADE_AnimateDiffLoaderWithContext" + }, + "widgets_values": [ + "mm_sd_v15_v2.ckpt", + "sqrt_linear (AnimateDiff)" + ] + }, + { + "id": 271, + "type": "VHS_VideoCombine", + "pos": [ + -510, + 100 + ], + "size": { + "0": 800, + "1": 980 + }, + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 436 + }, + { + "name": "filename_prefix", + "type": "STRING", + "link": 695, + "widget": { + "name": "filename_prefix", + "config": [ + "STRING", + { + "default": "AnimateDiff" + } + ] + } + }, + { + "name": "frame_rate", + "type": "INT", + "link": 699, + "widget": { + "name": "frame_rate", + "config": [ + "INT", + { + "default": 8, + "min": 1, + "step": 1 + } + ] + } + } + ], + "outputs": [ + { + "name": "GIF", + "type": "GIF", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": [ + 12, + 0, + "AnimateDiff/AD", + "image/gif", + false, + true + ] + }, + { + "id": 275, + "type": "ImpactKSamplerBasicPipe", + "pos": [ + -311.66975156067997, + -335.3463791916187 + ], + "size": { + "0": 315, + "1": 242 + }, + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "basic_pipe", + "type": "BASIC_PIPE", + "link": 454 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 446 + } + ], + "outputs": [ + { + "name": "BASIC_PIPE", + "type": "BASIC_PIPE", + "links": null, + "shape": 3 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 448 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 447 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "ImpactKSamplerBasicPipe" + }, + "widgets_values": [ + 153435234221149, + "randomize", + 25, + 7, + "dpmpp_sde_gpu", + "karras", + 1 + ] + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "pos": [ + -2870, + -420 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": { + "collapsed": true + }, + "order": 9, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 471 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 427, + 458, + 707 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 382 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "juggernaut_final.safetensors" + ] + }, + { + "id": 316, + "type": "LoadAudio", + "pos": [ + -6400, + -600 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 541, + 656 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "Real Good Snippet.wav", + "Audio" + ] + }, + { + "id": 405, + "type": "PreviewImage", + "pos": [ + -4930, + -700 + ], + "size": { + "0": 1950, + "1": 330 + }, + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 657 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 379, + "type": "PreviewImage", + "pos": [ + -4180, + -170 + ], + "size": { + "0": 1080, + "1": 290 + }, + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 626 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } } ], "links": [ @@ -2434,6 +2434,30 @@ "color": "#8A8", "font_size": 24, "locked": false + }, + { + "title": "Load Audio & Config", + "bounding": [ + -6425, + -694, + 736, + 346 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + }, + { + "title": "Preview Audio", + "bounding": [ + -5335, + -799, + 2378, + 470 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false } ], "config": {}, diff --git a/examples/basic_example.png b/examples/basic_example.png index f0349ed..2f2ddf8 100644 Binary files a/examples/basic_example.png and b/examples/basic_example.png differ diff --git a/nodes.py b/nodes.py index c142383..5dfc28d 100644 --- a/nodes.py +++ b/nodes.py @@ -216,6 +216,114 @@ class BatchAmplitudeSchedule: return (key_frame_series,) +class ClipAmplitude: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "amplitude": ("AMPLITUDE",), + "max_amplitude": ("INT", {"default": 1000, "min": 0, "step": 1}), + }, + "optional": { + "min_amplitude": ("INT", {"default": 0, "min": 0, "step": 1}), + } + } + + CATEGORY = "AudioScheduler/Amplitude" + + RETURN_TYPES = ("AMPLITUDE",) + RETURN_NAMES = ("amplitude",) + FUNCTION = "clip" + + def clip(self, amplitude, max_amplitude:int, min_amplitude:int=0): + if (min_amplitude > max_amplitude): + raise ValueError(f"min_amplitude '{min_amplitude}' cannot be higher than max_amplitude '{max_amplitude}'") + + clipped_amp = np.where(amplitude < max_amplitude, amplitude, max_amplitude) + clipped_amp = np.where(min_amplitude < clipped_amp, clipped_amp, min_amplitude) + return (clipped_amp,) + +class TransientAmplitudeBasic: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "amplitude": ("AMPLITUDE",), + }, + "optional": { + "frames_to_attack": ("INT", {"default": 0, "min": 0, "step": 1}), + "frames_to_hold": ("INT", {"default": 6, "min": 0, "step": 1}), + "frames_to_release": ("INT", {"default": 6, "min": 0, "step": 1}), + } + } + + CATEGORY = "AudioScheduler/Amplitude" + + RETURN_TYPES = ("AMPLITUDE",) + RETURN_NAMES = ("amplitude",) + FUNCTION = "adjust" + + def adjust(self, amplitude, frames_to_attack:int, frames_to_hold:int, frames_to_release:int): + if (frames_to_attack < 0): + raise ValueError(f"frames_to_attack '{frames_to_attack}' cannot be negative") + + if (frames_to_hold < 0): + raise ValueError(f"frames_to_hold '{frames_to_hold}' cannot be negative") + + if (frames_to_release < 0): + raise ValueError(f"frames_to_release '{frames_to_release}' cannot be negative") + + if (len(amplitude) <= 1): + return (amplitude,) + + if (frames_to_attack == 0 and frames_to_hold == 0 and frames_to_release == 0): + return (amplitude,) + + # Calculate the rise factor based on the number of frames to attack + rise_factor = 1 / (frames_to_attack + 1) + + # Calculate the decay factor based on the number of frames to release + decay_factor = 1 / (frames_to_release + 1) + + holding_frame = 0 + local_max_amplitude = amplitude[0] + prev_amplitude = amplitude[0] + adjusted_amp = [prev_amplitude] + for i in range(1, len(amplitude)): + + # attack + if (amplitude[i] >= prev_amplitude): + # reset and set new goal + holding_frame = 0 + local_max_amplitude = amplitude[i] + + # rise to the goal + if (frames_to_attack > 0): + prev_amplitude += local_max_amplitude * rise_factor + if (prev_amplitude > amplitude[i]): + prev_amplitude = amplitude[i] + else: + prev_amplitude = amplitude[i] + adjusted_amp.append(prev_amplitude) + continue + + # hold + if (frames_to_hold > 0 and holding_frame < frames_to_hold): + holding_frame += 1 + adjusted_amp.append(prev_amplitude) + continue + + # release + if (frames_to_release > 0): + prev_amplitude -= local_max_amplitude * decay_factor + if (prev_amplitude < amplitude[i]): + prev_amplitude = amplitude[i] + adjusted_amp.append(prev_amplitude) + continue + + # no adjustments for this frame + adjusted_amp.append(amplitude[i]) + + return (adjusted_amp,) + class NormalizeAmplitude: @classmethod def INPUT_TYPES(s): @@ -381,23 +489,31 @@ NODE_CLASS_MAPPINGS = { "LoadAudio": LoadAudio, "AudioToFFTs": AudioToFFTs, "AudioToAmplitudeGraph": AudioToAmplitudeGraph, + # Amplitude "BatchAmplitudeSchedule": BatchAmplitudeSchedule, + "ClipAmplitude": ClipAmplitude, + "TransientAmplitudeBasic": TransientAmplitudeBasic, + "AmplitudeToNumber" : AmplitudeToNumber, + "AmplitudeToGraph" : AmplitudeToGraph, + # Normalized Amplitude "NormalizeAmplitude": NormalizeAmplitude, "GateNormalizedAmplitude": GateNormalizedAmplitude, "NormalizedAmplitudeToNumber" : NormalizedAmplitudeToNumber, "NormalizedAmplitudeToGraph" : NormalizedAmplitudeToGraph, - "AmplitudeToNumber" : AmplitudeToNumber, - "AmplitudeToGraph" : AmplitudeToGraph, } NODE_DISPLAY_NAME_MAPPINGS = { "LoadAudio": "Load Audio", "AudioToFFTs": "Audio to FFTs", "AudioToAmplitudeGraph": "Audio to Amplitude Graph", + # Amplitude "BatchAmplitudeSchedule": "Batch Amplitude Schedule", + "ClipAmplitude": "Clip Amplitude", + "TransientAmplitudeBasic": "Transient Amplitude Basic", + "AmplitudeToNumber" : "Amplitude To Float or Int", + "AmplitudeToGraph" : "Amplitude To Graph", + # Normalized Amplitude "NormalizeAmplitude": "Normalize Amplitude", "GateNormalizedAmplitude": "Gate Normalized Amplitude", "NormalizedAmplitudeToNumber" : "Normalized Amplitude To Float or Int", "NormalizedAmplitudeToGraph" : "Normalized Amplitude To Graph", - "AmplitudeToNumber" : "Amplitude To Float or Int", - "AmplitudeToGraph" : "Amplitude To Graph", -} \ No newline at end of file +}