diff --git a/examples/ace1.5/audio_ace_step_1_5_cover.json b/examples/ace1.5/audio_ace_step_1_5_cover.json new file mode 100644 index 0000000..4b9c87e --- /dev/null +++ b/examples/ace1.5/audio_ace_step_1_5_cover.json @@ -0,0 +1 @@ +{"id":"5053cd4b-bb11-4361-b515-8844ef227a1c","revision":0,"last_node_id":29,"last_link_id":35,"nodes":[{"id":10,"type":"Knob","pos":[3180.335422078192,529.3368769156402],"size":[270,122],"flags":{},"order":17,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":10},{"localized_name":"knob","name":"knob","type":"FLOAT","widget":{"name":"knob"},"link":null},{"localized_name":"other_knob","name":"other_knob","type":"FLOAT","widget":{"name":"other_knob"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[34]}],"properties":{"cnr_id":"comfyui_ryanontheinside","ver":"951b1318f8958bfc441c0bcbbf2fc0695e1d9278","Node name for S&R":"Knob"},"widgets_values":[0.2,0.05]},{"id":12,"type":"PrimitiveNode","pos":[551.3403829650877,319.3290474243163],"size":[278,82],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[{"name":"INT","type":"INT","widget":{"name":"duration"},"links":[17]}],"title":"Song Duration","properties":{"Run widget replace on values":false},"widgets_values":[40,"fixed"]},{"id":13,"type":"ConditioningZeroOut","pos":[1180.3403829650877,579.3290474243163],"size":[204.134765625,26],"flags":{},"order":13,"mode":0,"inputs":[{"localized_name":"conditioning","name":"conditioning","type":"CONDITIONING","link":11}],"outputs":[{"localized_name":"CONDITIONING","name":"CONDITIONING","type":"CONDITIONING","links":[21]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"ConditioningZeroOut"},"widgets_values":[]},{"id":14,"type":"VAEEncodeAudio","pos":[1218.3403829650877,711.3290474243163],"size":[210,46],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":12},{"localized_name":"vae","name":"vae","type":"VAE","link":13}],"outputs":[{"localized_name":"LATENT","name":"LATENT","type":"LATENT","links":[22,29]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"VAEEncodeAudio"},"widgets_values":[]},{"id":15,"type":"CheckpointLoaderSimple","pos":[463.34038296508766,113.32904742431629],"size":[428,98],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"ckpt_name","name":"ckpt_name","type":"COMBO","widget":{"name":"ckpt_name"},"link":null}],"outputs":[{"localized_name":"MODEL","name":"MODEL","type":"MODEL","links":[24]},{"localized_name":"CLIP","name":"CLIP","type":"CLIP","links":[15]},{"localized_name":"VAE","name":"VAE","type":"VAE","links":[13,32]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"CheckpointLoaderSimple"},"widgets_values":["ace_step_1.5_turbo_aio.safetensors"]},{"id":16,"type":"VHS_LoadAudioUpload","pos":[259.34038296508766,484.3290474243163],"size":[252.0078125,204],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"start_time","name":"start_time","shape":7,"type":"FLOAT","widget":{"name":"start_time"},"link":null},{"localized_name":"duration","name":"duration","shape":7,"type":"FLOAT","widget":{"name":"duration"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[12,14,35]},{"localized_name":"duration","name":"duration","type":"FLOAT","links":null}],"properties":{"cnr_id":"comfyui-videohelpersuite","ver":"1.7.7","Node name for S&R":"VHS_LoadAudioUpload"},"widgets_values":{"audio":"Vesuvius_v2_edit.wav","start_time":0,"duration":60,"audiopreview":{"params":{"filename":"Vesuvius_v2_edit.wav","type":"input","start_time":0,"duration":60}}}},{"id":17,"type":"AudioInfo","pos":[561.3403829650877,474.3290474243163],"size":[270,378],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":14},{"localized_name":"frame_rate","name":"frame_rate","type":"FLOAT","widget":{"name":"frame_rate"},"link":null}],"outputs":[{"localized_name":"total_frames","name":"total_frames","type":"INT","links":null},{"localized_name":"frames_per_beat","name":"frames_per_beat","type":"INT","links":null},{"localized_name":"frames_per_bar","name":"frames_per_bar","type":"INT","links":null},{"localized_name":"frames_per_quarter","name":"frames_per_quarter","type":"INT","links":null},{"localized_name":"frames_per_eighth","name":"frames_per_eighth","type":"INT","links":null},{"localized_name":"audio_duration","name":"audio_duration","type":"FLOAT","links":null},{"localized_name":"beats_per_second","name":"beats_per_second","type":"FLOAT","links":null},{"localized_name":"detected_bpm","name":"detected_bpm","type":"FLOAT","links":[]},{"localized_name":"detected_bpm_rounded","name":"detected_bpm_rounded","type":"INT","links":[16]},{"localized_name":"sample_rate","name":"sample_rate","type":"INT","links":null},{"localized_name":"num_channels","name":"num_channels","type":"INT","links":null},{"localized_name":"num_samples","name":"num_samples","type":"INT","links":null},{"localized_name":"max_amplitude","name":"max_amplitude","type":"FLOAT","links":null},{"localized_name":"mean_amplitude","name":"mean_amplitude","type":"FLOAT","links":null},{"localized_name":"rms_amplitude","name":"rms_amplitude","type":"FLOAT","links":null},{"localized_name":"bit_depth","name":"bit_depth","type":"STRING","links":null},{"localized_name":"detected_key","name":"detected_key","type":"COMBO","links":[18]}],"properties":{"cnr_id":"comfyui_ryanontheinside","ver":"418dd269462f0f08096ae5ab899f991500f3a44c","Node name for S&R":"AudioInfo"},"widgets_values":[30]},{"id":19,"type":"ACEStep15NativeCoverGuider","pos":[2120.660994667053,294.4027658107756],"size":[359,146],"flags":{},"order":14,"mode":0,"inputs":[{"localized_name":"model","name":"model","type":"MODEL","link":19},{"localized_name":"positive","name":"positive","type":"CONDITIONING","link":20},{"localized_name":"negative","name":"negative","type":"CONDITIONING","link":21},{"localized_name":"source_latents","name":"source_latents","type":"LATENT","link":22},{"localized_name":"semantic_hints","name":"semantic_hints","shape":7,"type":"SEMANTIC_HINTS","link":null},{"localized_name":"cfg","name":"cfg","type":"FLOAT","widget":{"name":"cfg"},"link":null}],"outputs":[{"localized_name":"GUIDER","name":"GUIDER","type":"GUIDER","links":[26]}],"properties":{"cnr_id":"comfyui_ryanontheinside","ver":"418dd269462f0f08096ae5ab899f991500f3a44c","Node name for S&R":"ACEStep15NativeCoverGuider"},"widgets_values":[1]},{"id":20,"type":"KSamplerSelect","pos":[2160.660994667053,494.4027658107756],"size":[315,58],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"sampler_name","name":"sampler_name","type":"COMBO","widget":{"name":"sampler_name"},"link":null}],"outputs":[{"localized_name":"SAMPLER","name":"SAMPLER","type":"SAMPLER","links":[27]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"KSamplerSelect"},"widgets_values":["euler"]},{"id":21,"type":"BasicScheduler","pos":[2180.660994667053,674.4027658107756],"size":[270,106],"flags":{},"order":10,"mode":0,"inputs":[{"localized_name":"model","name":"model","type":"MODEL","link":23},{"localized_name":"scheduler","name":"scheduler","type":"COMBO","widget":{"name":"scheduler"},"link":null},{"localized_name":"steps","name":"steps","type":"INT","widget":{"name":"steps"},"link":null},{"localized_name":"denoise","name":"denoise","type":"FLOAT","widget":{"name":"denoise"},"link":null}],"outputs":[{"localized_name":"SIGMAS","name":"SIGMAS","type":"SIGMAS","links":[28]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"BasicScheduler"},"widgets_values":["simple",30,1]},{"id":22,"type":"RandomNoise","pos":[2130.660994667053,144.40276581077512],"size":[315,82],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"noise_seed","name":"noise_seed","type":"INT","widget":{"name":"noise_seed"},"link":null}],"outputs":[{"localized_name":"NOISE","name":"NOISE","type":"NOISE","links":[25]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"RandomNoise"},"widgets_values":[990,"fixed"]},{"id":23,"type":"ModelSamplingAuraFlow","pos":[1693.1675673904415,282.3494657161712],"size":[330,60],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"model","name":"model","type":"MODEL","link":24},{"localized_name":"shift","name":"shift","type":"FLOAT","widget":{"name":"shift"},"link":null}],"outputs":[{"localized_name":"MODEL","name":"MODEL","type":"MODEL","links":[19,23]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"ModelSamplingAuraFlow"},"widgets_values":[3]},{"id":24,"type":"SamplerCustomAdvanced","pos":[2693.727706092834,376.9360795619962],"size":[212.3638671875,106],"flags":{},"order":15,"mode":0,"inputs":[{"localized_name":"noise","name":"noise","type":"NOISE","link":25},{"localized_name":"guider","name":"guider","type":"GUIDER","link":26},{"localized_name":"sampler","name":"sampler","type":"SAMPLER","link":27},{"localized_name":"sigmas","name":"sigmas","type":"SIGMAS","link":28},{"localized_name":"latent_image","name":"latent_image","type":"LATENT","link":29}],"outputs":[{"localized_name":"output","name":"output","type":"LATENT","links":[]},{"localized_name":"denoised_output","name":"denoised_output","type":"LATENT","links":[31]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"SamplerCustomAdvanced"},"widgets_values":[]},{"id":25,"type":"PreviewAudio","pos":[3569.0547222900386,756.6726754379266],"size":[270,88],"flags":{},"order":12,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":30},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"PreviewAudio"},"widgets_values":[]},{"id":26,"type":"VAEDecodeAudio","pos":[3032.9211315917964,396.2059891891478],"size":[190,46],"flags":{},"order":16,"mode":0,"inputs":[{"localized_name":"samples","name":"samples","type":"LATENT","link":31},{"localized_name":"vae","name":"vae","type":"VAE","link":32}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[10,33]}],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"VAEDecodeAudio"},"widgets_values":[]},{"id":27,"type":"SaveAudioMP3","pos":[3288.7883801269527,319.6725495529174],"size":[424.16007385253897,147.58003692626949],"flags":{},"order":18,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":33},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"quality","name":"quality","type":"COMBO","widget":{"name":"quality"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"SaveAudioMP3"},"widgets_values":["audio/ace_step_cover","V0"]},{"id":28,"type":"PreviewAudio","pos":[3547.6547894287105,559.2726586532592],"size":[270,88],"flags":{},"order":19,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":34},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"cnr_id":"comfy-core","ver":"0.12.2","Node name for S&R":"PreviewAudio"},"widgets_values":[]},{"id":29,"type":"Reroute","pos":[3238.9883465576168,754.4726250839233],"size":[75,26],"flags":{},"order":9,"mode":0,"inputs":[{"name":"","type":"*","link":35}],"outputs":[{"name":"","type":"AUDIO","links":[30]}],"properties":{"showOutputText":false,"horizontal":false}},{"id":18,"type":"ACEStep15TaskTextEncode","pos":[985.3403829650877,149.3290474243163],"size":[400,358],"flags":{},"order":11,"mode":0,"inputs":[{"localized_name":"clip","name":"clip","type":"CLIP","link":15},{"localized_name":"text","name":"text","type":"STRING","widget":{"name":"text"},"link":null},{"localized_name":"task_type","name":"task_type","type":"COMBO","widget":{"name":"task_type"},"link":null},{"localized_name":"track_name","name":"track_name","shape":7,"type":"COMBO","widget":{"name":"track_name"},"link":null},{"localized_name":"lyrics","name":"lyrics","shape":7,"type":"STRING","widget":{"name":"lyrics"},"link":null},{"localized_name":"bpm","name":"bpm","shape":7,"type":"INT","widget":{"name":"bpm"},"link":16},{"localized_name":"duration","name":"duration","shape":7,"type":"INT","widget":{"name":"duration"},"link":17},{"localized_name":"keyscale","name":"keyscale","shape":7,"type":"COMBO","widget":{"name":"keyscale"},"link":18},{"localized_name":"timesignature","name":"timesignature","shape":7,"type":"COMBO","widget":{"name":"timesignature"},"link":null},{"localized_name":"language","name":"language","shape":7,"type":"COMBO","widget":{"name":"language"},"link":null},{"localized_name":"seed","name":"seed","shape":7,"type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"conditioning","name":"conditioning","type":"CONDITIONING","links":[11,20]}],"properties":{"cnr_id":"comfyui_ryanontheinside","ver":"418dd269462f0f08096ae5ab899f991500f3a44c","Node name for S&R":"ACEStep15TaskTextEncode"},"widgets_values":["jazz piano cover with swing rhythm","cover","","",120,40,"C major","4","en",0,"fixed"]},{"id":11,"type":"MarkdownNote","pos":[-6.659617034912344,73.32904742431629],"size":[420,360],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"properties":{},"widgets_values":["## ACE-Step 1.5 Cover Workflow\n\nAceStep1.5 Cover task\n\nGenerates new content across the entire duration while using the source audio structure as guidance.\n\n**Model:**\n- [ace_step_1.5_turbo_aio.safetensors](https://huggingface.co/Comfy-Org/ace_step_1.5_ComfyUI_files/resolve/main/checkpoints/ace_step_1.5_turbo_aio.safetensors)\n\n## [gimmi stars on github](https://github.com/ryanontheinside/ComfyUI_RyanOnTheInside)\n\n## [follow me on youtube](https://www.youtube.com/@ryanontheinside)"],"color":"#222","bgcolor":"#000"}],"links":[[10,26,0,10,0,"AUDIO"],[11,18,0,13,0,"CONDITIONING"],[12,16,0,14,0,"AUDIO"],[13,15,2,14,1,"VAE"],[14,16,0,17,0,"AUDIO"],[15,15,1,18,0,"CLIP"],[16,17,8,18,5,"INT"],[17,12,0,18,6,"INT"],[18,17,16,18,7,"COMBO"],[19,23,0,19,0,"MODEL"],[20,18,0,19,1,"CONDITIONING"],[21,13,0,19,2,"CONDITIONING"],[22,14,0,19,3,"LATENT"],[23,23,0,21,0,"MODEL"],[24,15,0,23,0,"MODEL"],[25,22,0,24,0,"NOISE"],[26,19,0,24,1,"GUIDER"],[27,20,0,24,2,"SAMPLER"],[28,21,0,24,3,"SIGMAS"],[29,14,0,24,4,"LATENT"],[30,29,0,25,0,"AUDIO"],[31,24,1,26,0,"LATENT"],[32,15,2,26,1,"VAE"],[33,26,0,27,0,"AUDIO"],[34,10,0,28,0,"AUDIO"],[35,16,0,29,0,"AUDIO"]],"groups":[{"id":1,"title":"Load Model","bounding":[453.34038296508766,43.329047424316286,448,182],"color":"#3f789e","font_size":24,"flags":{}},{"id":2,"title":"Duration","bounding":[453.34038296508766,243.3290474243163,450,170],"color":"#3f789e","font_size":24,"flags":{}},{"id":3,"title":"COVER","bounding":[933.3403829650877,43.329047424316286,640,820],"color":"#3f789e","font_size":24,"flags":{}},{"id":4,"title":"Guider","bounding":[1614.1331541109707,48.050058642859426,1329.466728210446,814.800117492674],"color":"#3f789e","font_size":24,"flags":{}},{"id":5,"title":"Group","bounding":[2980.685405293525,50.92808060521634,1011.9334564208966,839.0000839233378],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"ds":{"scale":1.6105100000000008,"offset":[277.4288227725874,24.50885306740408]},"workflowRendererVersion":"LG"},"version":0.4} \ No newline at end of file diff --git a/examples/ace1.5/audio_ace_step_1_5_edit.json b/examples/ace1.5/audio_ace_step_1_5_edit.json new file mode 100644 index 0000000..96ae9c7 --- /dev/null +++ b/examples/ace1.5/audio_ace_step_1_5_edit.json @@ -0,0 +1,1138 @@ +{ + "id": "1321ed79-5bbf-44b4-a7f2-cc6914f8ed3c", + "revision": 0, + "last_node_id": 18, + "last_link_id": 24, + "nodes": [ + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 271.1492814143214, + -75.31056421018627 + ], + "size": [ + 428, + 98 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 19 + ] + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 2 + ] + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 8, + 21 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "ace_step_1.5_turbo_aio.safetensors" + ] + }, + { + "id": 2, + "type": "PrimitiveNode", + "pos": [ + 359.1492814143214, + 130.68943578981373 + ], + "size": [ + 278, + 82 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "INT", + "type": "INT", + "widget": { + "name": "duration" + }, + "links": [ + 4 + ] + } + ], + "title": "Song Duration (total after edit)", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 60, + "fixed" + ] + }, + { + "id": 6, + "type": "VAEEncodeAudio", + "pos": [ + 1150, + 700 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 13 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "VAEEncodeAudio" + }, + "widgets_values": [] + }, + { + "id": 7, + "type": "BasicScheduler", + "pos": [ + 1871.1492814143214, + 334.68943578981373 + ], + "size": [ + 270, + 106 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 9 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 17 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "simple", + 30, + 1 + ] + }, + { + "id": 9, + "type": "KSamplerSelect", + "pos": [ + 1871.1492814143214, + 204.68943578981373 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 16 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "euler" + ] + }, + { + "id": 12, + "type": "RandomNoise", + "pos": [ + 1881.1492814143214, + 84.68943578981373 + ], + "size": [ + 315, + 82 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 14 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 990, + "fixed" + ] + }, + { + "id": 13, + "type": "ModelSamplingAuraFlow", + "pos": [ + 1491.5184221515126, + 124.43989184642457 + ], + "size": [ + 330, + 60 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 19 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 9, + 10 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "ModelSamplingAuraFlow" + }, + "widgets_values": [ + 3 + ] + }, + { + "id": 5, + "type": "ConditioningZeroOut", + "pos": [ + 1546.6533669910605, + 572.241559278368 + ], + "size": [ + 204.134765625, + 26 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 6 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 12 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "ConditioningZeroOut" + }, + "widgets_values": [] + }, + { + "id": 15, + "type": "Knob", + "pos": [ + 2774.678850395049, + 285.9556615665655 + ], + "size": [ + 270, + 122 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 22 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 24 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "951b1318f8958bfc441c0bcbbf2fc0695e1d9278", + "Node name for S&R": "Knob" + }, + "widgets_values": [ + 0.2, + 0.05 + ] + }, + { + "id": 11, + "type": "MarkdownNote", + "pos": [ + -236.51998657413333, + -185.68166328106963 + ], + "size": [ + 420, + 400 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "## ACE-Step 1.5 Edit Workflow\n\nUnified workflow for extend and/or repaint operations.\n\n**Extend:** Add content before/after audio\n- Set extend_left_seconds and/or extend_right_seconds > 0\n\n**Repaint:** Regenerate a section\n- Set repaint_start_seconds >= 0 and repaint_end_seconds > start\n\n**Both:** Combine extend and repaint in one pass\n\n**Model:**\n- [ace_step_1.5_turbo_aio.safetensors](https://huggingface.co/Comfy-Org/ace_step_1.5_ComfyUI_files/resolve/main/checkpoints/ace_step_1.5_turbo_aio.safetensors)\n\n## [gimmi stars on github](https://github.com/ryanontheinside/ComfyUI_RyanOnTheInside)\n\n## [follow me on youtube](https://www.youtube.com/@ryanontheinside)\n" + ], + "color": "#222", + "bgcolor": "#000" + }, + { + "id": 3, + "type": "AudioInfo", + "pos": [ + 349.1010256576013, + 314.1789956585514 + ], + "size": [ + 270, + 378 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 1 + } + ], + "outputs": [ + { + "name": "total_frames", + "type": "INT", + "links": null + }, + { + "name": "frames_per_beat", + "type": "INT", + "links": null + }, + { + "name": "frames_per_bar", + "type": "INT", + "links": null + }, + { + "name": "frames_per_quarter", + "type": "INT", + "links": null + }, + { + "name": "frames_per_eighth", + "type": "INT", + "links": null + }, + { + "name": "audio_duration", + "type": "FLOAT", + "links": null + }, + { + "name": "beats_per_second", + "type": "FLOAT", + "links": null + }, + { + "name": "detected_bpm", + "type": "FLOAT", + "links": [] + }, + { + "name": "detected_bpm_rounded", + "type": "INT", + "links": [ + 3 + ] + }, + { + "name": "sample_rate", + "type": "INT", + "links": null + }, + { + "name": "num_channels", + "type": "INT", + "links": null + }, + { + "name": "num_samples", + "type": "INT", + "links": null + }, + { + "name": "max_amplitude", + "type": "FLOAT", + "links": null + }, + { + "name": "mean_amplitude", + "type": "FLOAT", + "links": null + }, + { + "name": "rms_amplitude", + "type": "FLOAT", + "links": null + }, + { + "name": "bit_depth", + "type": "STRING", + "links": null + }, + { + "name": "detected_key", + "type": "COMBO", + "links": [ + 5 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "418dd269462f0f08096ae5ab899f991500f3a44c", + "Node name for S&R": "AudioInfo" + }, + "widgets_values": [ + 30 + ] + }, + { + "id": 17, + "type": "PreviewAudio", + "pos": [ + 3124.678850395049, + 485.9556615665655 + ], + "size": [ + 270, + 88 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 23 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [] + }, + { + "id": 4, + "type": "ACEStep15TaskTextEncode", + "pos": [ + 830, + 200 + ], + "size": [ + 400, + 358 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + }, + { + "name": "bpm", + "shape": 7, + "type": "INT", + "widget": { + "name": "bpm" + }, + "link": 3 + }, + { + "name": "duration", + "shape": 7, + "type": "INT", + "widget": { + "name": "duration" + }, + "link": 4 + }, + { + "name": "keyscale", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "keyscale" + }, + "link": 5 + } + ], + "outputs": [ + { + "name": "conditioning", + "type": "CONDITIONING", + "links": [ + 6, + 11 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "418dd269462f0f08096ae5ab899f991500f3a44c", + "Node name for S&R": "ACEStep15TaskTextEncode" + }, + "widgets_values": [ + "continue the melodic electronic track with soft synths", + "repaint", + "", + "", + 120, + 60, + "C major", + "4", + "en", + 0, + "fixed" + ] + }, + { + "id": 16, + "type": "VHS_LoadAudioUpload", + "pos": [ + 65.03891010326599, + 377.9928905051513 + ], + "size": [ + 252.0078125, + 204 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 1, + 7, + 23 + ] + }, + { + "name": "duration", + "type": "FLOAT", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-videohelpersuite", + "ver": "1.7.7", + "Node name for S&R": "VHS_LoadAudioUpload" + }, + "widgets_values": { + "audio": "Vesuvius_v2_edit.wav", + "start_time": 25, + "duration": 40, + "audiopreview": { + "params": { + "filename": "Vesuvius_v2_edit.wav", + "type": "input", + "start_time": 25, + "duration": 40 + } + } + } + }, + { + "id": 8, + "type": "ACEStep15NativeEditGuider", + "pos": [ + 1841.1492814143214, + 504.68943578981373 + ], + "size": [ + 449.1958984375, + 214 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 10 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 11 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 12 + }, + { + "name": "source_latents", + "type": "LATENT", + "link": 13 + } + ], + "outputs": [ + { + "name": "guider", + "type": "GUIDER", + "links": [ + 15 + ] + }, + { + "name": "output_latent", + "type": "LATENT", + "links": [ + 18 + ] + } + ], + "properties": { + "cnr_id": "comfyui_ryanontheinside", + "ver": "98474ec06548c527400ca866ca65f2b4c477712b", + "Node name for S&R": "ACEStep15NativeEditGuider" + }, + "widgets_values": [ + 1, + 10, + 20, + -1, + -1 + ] + }, + { + "id": 18, + "type": "SaveAudioMP3", + "pos": [ + 3104.678850395049, + 225.9556615665654 + ], + "size": [ + 550, + 150 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 24 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "SaveAudioMP3" + }, + "widgets_values": [ + "audio/ace_step_edit", + "V0" + ] + }, + { + "id": 10, + "type": "SamplerCustomAdvanced", + "pos": [ + 2327.3039309130986, + 220.2170683486411 + ], + "size": [ + 212.3638671875, + 106 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 14 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 15 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 16 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 17 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 18 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [] + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [ + 20 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "SamplerCustomAdvanced" + }, + "widgets_values": [] + }, + { + "id": 14, + "type": "VAEDecodeAudio", + "pos": [ + 2774.678850395049, + 145.9556615665654 + ], + "size": [ + 190, + 46 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 20 + }, + { + "name": "vae", + "type": "VAE", + "link": 21 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 22 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.12.2", + "Node name for S&R": "VAEDecodeAudio" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 1, + 16, + 0, + 3, + 0, + "AUDIO" + ], + [ + 2, + 1, + 1, + 4, + 0, + "CLIP" + ], + [ + 3, + 3, + 8, + 4, + 1, + "INT" + ], + [ + 4, + 2, + 0, + 4, + 2, + "INT" + ], + [ + 5, + 3, + 16, + 4, + 3, + "COMBO" + ], + [ + 6, + 4, + 0, + 5, + 0, + "CONDITIONING" + ], + [ + 7, + 16, + 0, + 6, + 0, + "AUDIO" + ], + [ + 8, + 1, + 2, + 6, + 1, + "VAE" + ], + [ + 9, + 13, + 0, + 7, + 0, + "MODEL" + ], + [ + 10, + 13, + 0, + 8, + 0, + "MODEL" + ], + [ + 11, + 4, + 0, + 8, + 1, + "CONDITIONING" + ], + [ + 12, + 5, + 0, + 8, + 2, + "CONDITIONING" + ], + [ + 13, + 6, + 0, + 8, + 3, + "LATENT" + ], + [ + 14, + 12, + 0, + 10, + 0, + "NOISE" + ], + [ + 15, + 8, + 0, + 10, + 1, + "GUIDER" + ], + [ + 16, + 9, + 0, + 10, + 2, + "SAMPLER" + ], + [ + 17, + 7, + 0, + 10, + 3, + "SIGMAS" + ], + [ + 18, + 8, + 1, + 10, + 4, + "LATENT" + ], + [ + 19, + 1, + 0, + 13, + 0, + "MODEL" + ], + [ + 20, + 10, + 1, + 14, + 0, + "LATENT" + ], + [ + 21, + 1, + 2, + 14, + 1, + "VAE" + ], + [ + 22, + 14, + 0, + 15, + 0, + "AUDIO" + ], + [ + 23, + 16, + 0, + 17, + 0, + "AUDIO" + ], + [ + 24, + 15, + 0, + 18, + 0, + "AUDIO" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 261.1492814143214, + -145.31056421018627, + 448, + 182 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Duration", + "bounding": [ + 261.1492814143214, + 54.68943578981373, + 450, + 170 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "EXTEND REPAINT", + "bounding": [ + 764.4417814143217, + 6.645286047686568, + 640, + 820 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Guider", + "bounding": [ + 1445.8449100323544, + 15.291210910605969, + 1231.473021774977, + 794.3779944148237 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Output", + "bounding": [ + 2707.803145405084, + 20.763913715067083, + 1231.473021774977, + 794.3779944148237 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.4177248169415674, + "offset": [ + 605.3831100907462, + 1103.4212377905055 + ] + }, + "workflowRendererVersion": "LG", + "frontendVersion": "1.37.11", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes/__init__.py b/nodes/__init__.py new file mode 100644 index 0000000..890a2b2 --- /dev/null +++ b/nodes/__init__.py @@ -0,0 +1 @@ +# Empty init file for nodes package diff --git a/nodes/acestep/__init__.py b/nodes/acestep/__init__.py new file mode 100644 index 0000000..436cde6 --- /dev/null +++ b/nodes/acestep/__init__.py @@ -0,0 +1,26 @@ +""" +ACE-Step nodes for ComfyUI. + +This package provides nodes for ACE-Step 1.0 and 1.5 audio generation, +including extend, repaint, cover, and extract operations. +""" + +import logging +import os + +# Create logger for the acestep module +logger = logging.getLogger("comfyui_ryanontheinside.acestep") + +# Set default level from environment variable or default to INFO +# Use ACESTEP_LOG_LEVEL=DEBUG for verbose output +log_level = os.environ.get("ACESTEP_LOG_LEVEL", "INFO").upper() +logger.setLevel(getattr(logging, log_level, logging.INFO)) + +# Only add handler if none exist (avoid duplicate handlers on reload) +if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("[%(name)s] %(message)s")) + logger.addHandler(handler) + +# Prevent propagation to root logger to avoid duplicate messages +logger.propagate = False diff --git a/nodes/acestep/ace_step_guiders.py b/nodes/acestep/ace_step_guiders.py index 839a181..4055e27 100644 --- a/nodes/acestep/ace_step_guiders.py +++ b/nodes/acestep/ace_step_guiders.py @@ -3,46 +3,739 @@ import comfy.samplers import comfy.sample import math from .ace_step_utils import ACEStepLatentUtils +from . import logger # NOTE: gigantic shoutout to the powerful acestep team https://github.com/ace-step/ACE-Step # NOTE: And another massive shoutout.... the adapted code here is based on https://github.com/billwuhao/ComfyUI_ACE-Step # NOTE: this implementation is experimental and beta with a minimum of testing + +# ============================================================================= +# ACE-Step 1.5 Native Edit Guider (unified extend + repaint) +# ============================================================================= + +class ACEStep15NativeEditGuider(comfy.samplers.CFGGuider): + """ + Unified ACE-Step 1.5 edit guider that handles both extend and repaint operations. + + This guider uses the model's native mask input mechanism, injecting chunk_masks + and src_latents directly into the model's forward() method via model wrapping. + + Operations: + - Extend: Add new content before (left) and/or after (right) the source audio + - Repaint: Regenerate a specific time region within the audio + + Both operations can be combined in a single pass. + + The silence_latent is automatically loaded internally (downloaded from HuggingFace + on first use). It's a learned tensor that tells the model to generate new content. + """ + + def __init__(self, model, positive, negative, cfg, source_latent, + extend_left_seconds=0.0, extend_right_seconds=0.0, + repaint_start_seconds=None, repaint_end_seconds=None): + super().__init__(model) + self.set_conds(positive, negative) + self.set_cfg(cfg) + + self.source_latent = source_latent + + # Load silence_latent internally + from .ace_step_utils import load_silence_latent + silence_latent = load_silence_latent(verbose=True) + + # v1.5 frame rate: 48000 / 1920 = 25 fps + self.fps = 25.0 + + # Extension parameters + self.extend_left_seconds = extend_left_seconds + self.extend_right_seconds = extend_right_seconds + self.left_frames = int(extend_left_seconds * self.fps) + self.right_frames = int(extend_right_seconds * self.fps) + + batch_size, channels, source_length = source_latent.shape + total_length = self.left_frames + source_length + self.right_frames + + # Repaint parameters (adjusted for extension offset) + self.has_repaint = (repaint_start_seconds is not None and + repaint_end_seconds is not None and + repaint_end_seconds > repaint_start_seconds) + + if self.has_repaint: + # Adjust repaint region for left extension offset + adjusted_start = repaint_start_seconds + extend_left_seconds + adjusted_end = repaint_end_seconds + extend_left_seconds + self.repaint_start_frame = int(adjusted_start * self.fps) + self.repaint_end_frame = int(adjusted_end * self.fps) + # Clamp to valid range + self.repaint_start_frame = max(0, min(self.repaint_start_frame, total_length)) + self.repaint_end_frame = max(0, min(self.repaint_end_frame, total_length)) + else: + self.repaint_start_frame = 0 + self.repaint_end_frame = 0 + + logger.debug(f"[ACE15_EDIT] Initializing") + logger.debug(f"[ACE15_EDIT] source_latent.shape: {source_latent.shape}") + logger.debug(f"[ACE15_EDIT] silence_latent.shape: {silence_latent.shape}") + logger.debug(f"[ACE15_EDIT] extend: left={extend_left_seconds}s ({self.left_frames}f), right={extend_right_seconds}s ({self.right_frames}f)") + if self.has_repaint: + logger.debug(f"[ACE15_EDIT] repaint: {repaint_start_seconds}s-{repaint_end_seconds}s (frames {self.repaint_start_frame}-{self.repaint_end_frame} in extended space)") + logger.debug(f"[ACE15_EDIT] total_length: {total_length}") + + # Step 1: Prepare silence_latent in ComfyUI format [D, total_length] + # silence_latent shape is [1, T, D], need [D, total_length] for ComfyUI format + silence_tiled = silence_latent[0, :total_length, :] # [total_length, D] or less + if silence_tiled.shape[0] < total_length: + # Need to tile/repeat + num_tiles = (total_length // silence_tiled.shape[0]) + 1 + silence_tiled = silence_latent[0].repeat(num_tiles, 1)[:total_length, :] + # Transpose to [D, total_length] to match latent channel-first format + silence_tiled = silence_tiled.transpose(0, 1).to( + device=source_latent.device, dtype=source_latent.dtype + ) # [D, total_length] + + # Step 2: Create working latent + # IMPORTANT: In the reference, zero audio → VAE encode → silence_latent + # So extend regions should have silence_latent, not zeros + if self.left_frames > 0 or self.right_frames > 0: + # Initialize with silence_latent (matching reference behavior) + self.working_latent = silence_tiled.unsqueeze(0).expand(batch_size, -1, -1).clone() + # Copy source into the correct position (overwriting silence in source region) + self.working_latent[:, :, self.left_frames:self.left_frames + source_length] = source_latent + else: + self.working_latent = source_latent.clone() + + # Step 3: Create chunk_masks (0 = preserve, 1 = generate) + self.chunk_masks = torch.zeros_like(self.working_latent) + + # Mark extension regions for generation + if self.left_frames > 0: + self.chunk_masks[:, :, :self.left_frames] = 1.0 + if self.right_frames > 0: + self.chunk_masks[:, :, -self.right_frames:] = 1.0 + + # Mark repaint region for generation + if self.has_repaint: + self.chunk_masks[:, :, self.repaint_start_frame:self.repaint_end_frame] = 1.0 + + # Step 4: Create src_latents with silence_latent in generation regions + # Start from working_latent (which already has silence in extend regions) + self.src_latents = self.working_latent.clone() + + # Fill repaint region with silence_latent (extend regions already have it from working_latent) + # Use same indices as reference: src_latent[start:end] = silence_latent[start:end] + if self.has_repaint: + silence_repaint = silence_tiled[:, self.repaint_start_frame:self.repaint_end_frame] + self.src_latents[:, :, self.repaint_start_frame:self.repaint_end_frame] = silence_repaint.unsqueeze(0).expand(batch_size, -1, -1) + + # Log summary + gen_frames = self.chunk_masks.sum().item() / (batch_size * channels) + logger.debug(f"[ACE15_EDIT] silence_tiled.shape: {silence_tiled.shape}") + logger.debug(f"[ACE15_EDIT] working_latent.shape: {self.working_latent.shape}") + logger.debug(f"[ACE15_EDIT] src_latents.shape: {self.src_latents.shape}") + logger.debug(f"[ACE15_EDIT] chunk_masks.shape: {self.chunk_masks.shape}") + logger.debug(f"[ACE15_EDIT] generation frames: {gen_frames:.0f} ({gen_frames / self.fps:.2f}s)") + + # Verify chunk_masks has generation regions marked + mask_max = self.chunk_masks.max().item() + mask_min = self.chunk_masks.min().item() + logger.debug(f"[ACE15_EDIT] chunk_masks range: [{mask_min:.1f}, {mask_max:.1f}] (should have 1.0 for generation)") + + # Verify silence_latent was inserted (check mean/std of extend regions vs source region) + silence_ref_mean = silence_tiled.mean().item() + silence_ref_std = silence_tiled.std().item() + logger.debug(f"[ACE15_EDIT] silence_latent reference: mean={silence_ref_mean:.4f}, std={silence_ref_std:.4f}") + + if self.left_frames > 0: + left_mean = self.working_latent[:, :, :self.left_frames].mean().item() + left_std = self.working_latent[:, :, :self.left_frames].std().item() + src_left_mean = self.src_latents[:, :, :self.left_frames].mean().item() + logger.debug(f"[ACE15_EDIT] left_extend working_latent: mean={left_mean:.4f}, std={left_std:.4f}") + logger.debug(f"[ACE15_EDIT] left_extend src_latents: mean={src_left_mean:.4f} (should match silence_latent)") + if self.right_frames > 0: + right_mean = self.working_latent[:, :, -self.right_frames:].mean().item() + right_std = self.working_latent[:, :, -self.right_frames:].std().item() + src_right_mean = self.src_latents[:, :, -self.right_frames:].mean().item() + logger.debug(f"[ACE15_EDIT] right_extend working_latent: mean={right_mean:.4f}, std={right_std:.4f}") + logger.debug(f"[ACE15_EDIT] right_extend src_latents: mean={src_right_mean:.4f} (should match silence_latent)") + source_mean = self.working_latent[:, :, self.left_frames:self.left_frames + source_length].mean().item() + source_std = self.working_latent[:, :, self.left_frames:self.left_frames + source_length].std().item() + logger.debug(f"[ACE15_EDIT] source region: mean={source_mean:.4f}, std={source_std:.4f}") + + # Warn if no generation regions + if gen_frames == 0: + logger.debug(f"[ACE15_EDIT] WARNING: No generation frames! This will produce silence in extend regions.") + + self._wrapper_applied = False + + def _apply_model_wrapper(self): + """Apply model wrapper to inject chunk_masks and src_latents""" + if self._wrapper_applied: + return + + chunk_masks = self.chunk_masks + src_latents = self.src_latents + + # Clear any existing wrapper to avoid stale closure values + self.model_patcher.set_model_unet_function_wrapper(None) + + def model_function_wrapper(apply_model, args): + c = args["c"].copy() + device = args["input"].device + dtype = args["input"].dtype + + # Handle CFG batching + input_batch_size = args["input"].shape[0] + cm = chunk_masks.to(device=device, dtype=dtype) + sl = src_latents.to(device=device, dtype=dtype) + if cm.shape[0] < input_batch_size: + cm = cm.repeat(input_batch_size, 1, 1) + sl = sl.repeat(input_batch_size, 1, 1) + + c["chunk_masks"] = cm + c["src_latents"] = sl + # Explicitly set is_covers=0 for extend/repaint tasks + # This tells the model to use src_latents directly (not semantic hints) + c["is_covers"] = torch.zeros((input_batch_size,), device=device, dtype=torch.long) + + if not hasattr(model_function_wrapper, '_logged'): + logger.debug(f"[ACE15_EDIT_WRAPPER] Injecting chunk_masks and src_latents") + logger.debug(f"[ACE15_EDIT_WRAPPER] input.shape: {args['input'].shape}") + logger.debug(f"[ACE15_EDIT_WRAPPER] chunk_masks.shape: {c['chunk_masks'].shape}") + logger.debug(f"[ACE15_EDIT_WRAPPER] src_latents.shape: {c['src_latents'].shape}") + logger.debug(f"[ACE15_EDIT_WRAPPER] is_covers: {c['is_covers']} (0=extend/repaint mode)") + logger.debug(f"[ACE15_EDIT_WRAPPER] chunk_masks sum: {cm.sum().item():.0f} (generation frames * batch * channels)") + model_function_wrapper._logged = True + + return apply_model(args["input"], args["timestep"], **c) + + model_function_wrapper._logged = False + self.model_patcher.set_model_unet_function_wrapper(model_function_wrapper) + self._wrapper_applied = True + logger.debug(f"[ACE15_EDIT] Model wrapper applied") + + def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): + """Override sample to apply model wrapper and use working latent""" + + logger.debug(f"[ACE15_EDIT] sample() called") + logger.debug(f"[ACE15_EDIT] noise.shape: {noise.shape}") + logger.debug(f"[ACE15_EDIT] latent_image.shape: {latent_image.shape}") + + self._apply_model_wrapper() + + # Generate noise for working latent shape + device = noise.device + dtype = noise.dtype + working_noise = torch.randn_like(self.working_latent, device=device, dtype=dtype) + + logger.debug(f"[ACE15_EDIT] working_noise.shape: {working_noise.shape}") + logger.debug(f"[ACE15_EDIT] working_latent.shape: {self.working_latent.shape}") + + result = super().sample( + working_noise, + self.working_latent, + sampler, + sigmas, + denoise_mask, + callback, + disable_pbar, + seed + ) + + logger.debug(f"[ACE15_EDIT] result.shape: {result.shape}") + + return result + + +# Backwards compatibility aliases +ACEStep15NativeExtendGuider = ACEStep15NativeEditGuider +ACEStep15NativeRepaintGuider = ACEStep15NativeEditGuider + + +class ACEStep15NativeCoverGuider(comfy.samplers.CFGGuider): + """ + ACE-Step 1.5 cover guider for style transfer/regeneration. + + Uses semantic tokens (lm_hints) from the source audio as structural guidance + while generating new content with a different style/timbre. + + Official behavior: + - chunk_masks: All 1s (generate everything) + - is_covers: 1 (use semantic hints instead of raw latents) + - precomputed_lm_hints_25Hz: Semantic representation of source audio + + If semantic_hints is not provided, they will be automatically extracted from + the source_latent. For advanced workflows (e.g., blending hints from multiple + sources), use ACEStep15SemanticExtractor and ACEStep15SemanticHintsBlend. + """ + + def __init__(self, model, positive, negative, cfg, source_latent, semantic_hints=None): + super().__init__(model) + self.set_conds(positive, negative) + self.set_cfg(cfg) + + self.source_latent = source_latent + + batch_size, channels, total_length = source_latent.shape + + logger.debug(f"[ACE15_NATIVE_COVER] Initializing") + logger.debug(f"[ACE15_NATIVE_COVER] source_latent.shape: {source_latent.shape}") + logger.debug(f"[ACE15_NATIVE_COVER] source_latent stats: mean={source_latent.mean():.4f}, std={source_latent.std():.4f}") + + # Auto-extract semantic hints if not provided + if semantic_hints is None: + logger.debug(f"[ACE15_NATIVE_COVER] No semantic_hints provided, auto-extracting...") + from .ace_step_utils import extract_semantic_hints + semantic_hints = extract_semantic_hints(model, source_latent, verbose=True) + logger.debug(f"[ACE15_NATIVE_COVER] Auto-extracted semantic_hints shape: {semantic_hints.shape}") + + self.semantic_hints = semantic_hints + + # Validate semantic hints + logger.debug(f"[ACE15_NATIVE_COVER] semantic_hints stats: mean={semantic_hints.mean():.4f}, std={semantic_hints.std():.4f}") + if semantic_hints.std() < 0.01: + logger.debug(f"[ACE15_NATIVE_COVER] WARNING: semantic_hints have very low variance ({semantic_hints.std():.6f}) - may be invalid!") + if semantic_hints.shape != source_latent.shape: + logger.debug(f"[ACE15_NATIVE_COVER] WARNING: semantic_hints shape {semantic_hints.shape} != source_latent shape {source_latent.shape}") + + # Cover: generate everything + # chunk_masks: All 1s (generate all frames) + self.chunk_masks = torch.ones_like(source_latent) + + # src_latents: Original audio (used as fallback if is_covers=0) + self.src_latents = source_latent.clone() + + logger.debug(f"[ACE15_NATIVE_COVER] chunk_masks.shape: {self.chunk_masks.shape}") + logger.debug(f"[ACE15_NATIVE_COVER] mask sum: {self.chunk_masks.sum().item()} (all 1s)") + + self._wrapper_applied = False + + def _apply_model_wrapper(self): + """Apply model wrapper to inject chunk_masks, src_latents, is_covers, and semantic hints""" + if self._wrapper_applied: + return + + chunk_masks = self.chunk_masks + src_latents = self.src_latents + semantic_hints = self.semantic_hints + + self.model_patcher.set_model_unet_function_wrapper(None) + + def model_function_wrapper(apply_model, args): + c = args["c"].copy() + device = args["input"].device + dtype = args["input"].dtype + + input_batch_size = args["input"].shape[0] + cm = chunk_masks.to(device=device, dtype=dtype) + sl = src_latents.to(device=device, dtype=dtype) + if cm.shape[0] < input_batch_size: + cm = cm.repeat(input_batch_size, 1, 1) + sl = sl.repeat(input_batch_size, 1, 1) + + c["chunk_masks"] = cm + c["src_latents"] = sl + + # Set is_covers based on whether we have semantic hints + if semantic_hints is not None: + # Proper cover: use semantic hints via is_covers=1 + sh = semantic_hints.to(device=device, dtype=dtype) + if sh.shape[0] < input_batch_size: + sh = sh.repeat(input_batch_size, 1, 1) + c["precomputed_lm_hints_25Hz"] = sh + c["is_covers"] = torch.ones((input_batch_size,), device=device, dtype=torch.long) + else: + # Fallback: use raw VAE latents via is_covers=0 + c["is_covers"] = torch.zeros((input_batch_size,), device=device, dtype=torch.long) + + if not hasattr(model_function_wrapper, '_logged'): + logger.debug(f"[ACE15_COVER_WRAPPER] Injecting cover parameters") + logger.debug(f"[ACE15_COVER_WRAPPER] input.shape: {args['input'].shape}") + logger.debug(f"[ACE15_COVER_WRAPPER] input stats: mean={args['input'].mean():.4f}, std={args['input'].std():.4f}") + logger.debug(f"[ACE15_COVER_WRAPPER] chunk_masks.shape: {c['chunk_masks'].shape}") + logger.debug(f"[ACE15_COVER_WRAPPER] chunk_masks sum: {c['chunk_masks'].sum().item()} (should be all 1s for cover)") + logger.debug(f"[ACE15_COVER_WRAPPER] src_latents.shape: {c['src_latents'].shape}") + logger.debug(f"[ACE15_COVER_WRAPPER] src_latents stats: mean={c['src_latents'].mean():.4f}, std={c['src_latents'].std():.4f}") + logger.debug(f"[ACE15_COVER_WRAPPER] is_covers: {c['is_covers']} (1=use semantic hints, 0=use VAE latents)") + if "precomputed_lm_hints_25Hz" in c: + sh = c["precomputed_lm_hints_25Hz"] + logger.debug(f"[ACE15_COVER_WRAPPER] precomputed_lm_hints_25Hz.shape: {sh.shape}") + logger.debug(f"[ACE15_COVER_WRAPPER] precomputed_lm_hints_25Hz stats: mean={sh.mean():.4f}, std={sh.std():.4f}, min={sh.min():.4f}, max={sh.max():.4f}") + # Check if hints look valid (should have some variance, not constant) + if sh.std() < 0.01: + logger.debug(f"[ACE15_COVER_WRAPPER] WARNING: semantic hints have very low variance! May be constant/invalid.") + else: + logger.debug(f"[ACE15_COVER_WRAPPER] WARNING: No precomputed_lm_hints_25Hz - will fall back to VAE latents!") + model_function_wrapper._logged = True + + return apply_model(args["input"], args["timestep"], **c) + + model_function_wrapper._logged = False + self.model_patcher.set_model_unet_function_wrapper(model_function_wrapper) + self._wrapper_applied = True + logger.debug(f"[ACE15_NATIVE_COVER] Model wrapper applied") + + def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): + """Override sample to apply model wrapper for cover""" + + logger.debug(f"[ACE15_NATIVE_COVER] sample() called") + logger.debug(f"[ACE15_NATIVE_COVER] noise.shape: {noise.shape}") + logger.debug(f"[ACE15_NATIVE_COVER] latent_image.shape: {latent_image.shape}") + + self._apply_model_wrapper() + + device = noise.device + dtype = noise.dtype + working_noise = torch.randn_like(self.source_latent, device=device, dtype=dtype) + + result = super().sample( + working_noise, + self.source_latent, + sampler, + sigmas, + denoise_mask, + callback, + disable_pbar, + seed + ) + + logger.debug(f"[ACE15_NATIVE_COVER] result.shape: {result.shape}") + + return result + + +class ACEStep15NativeExtractGuider(comfy.samplers.CFGGuider): + """ + ACE-Step 1.5 extract guider for extracting specific tracks. + + Extracts a specific track (e.g., vocals, drums, bass) from the source audio. + The track_name is used in the text encoder instruction to guide extraction. + + Official behavior: + - chunk_masks: All 1s (generate everything) + - is_covers: 1 (use semantic hints for structure) + - precomputed_lm_hints_25Hz: Semantic representation of source audio + - Instruction: "Extract the {TRACK_NAME} track from the audio:" + + If semantic_hints is not provided, they will be automatically extracted from + the source_latent. For advanced workflows, use ACEStep15SemanticExtractor. + """ + + def __init__(self, model, positive, negative, cfg, source_latent, track_name="vocals", semantic_hints=None): + super().__init__(model) + self.set_conds(positive, negative) + self.set_cfg(cfg) + + self.source_latent = source_latent + self.track_name = track_name + + batch_size, channels, total_length = source_latent.shape + + logger.debug(f"[ACE15_NATIVE_EXTRACT] Initializing") + logger.debug(f"[ACE15_NATIVE_EXTRACT] source_latent.shape: {source_latent.shape}") + logger.debug(f"[ACE15_NATIVE_EXTRACT] track_name: {track_name}") + + # Auto-extract semantic hints if not provided + if semantic_hints is None: + logger.debug(f"[ACE15_NATIVE_EXTRACT] No semantic_hints provided, auto-extracting...") + from .ace_step_utils import extract_semantic_hints + semantic_hints = extract_semantic_hints(model, source_latent, verbose=True) + logger.debug(f"[ACE15_NATIVE_EXTRACT] Auto-extracted semantic_hints shape: {semantic_hints.shape}") + + self.semantic_hints = semantic_hints + + # Validate semantic hints + logger.debug(f"[ACE15_NATIVE_EXTRACT] semantic_hints stats: mean={semantic_hints.mean():.4f}, std={semantic_hints.std():.4f}") + + # Extract: generate everything (the extracted track) + self.chunk_masks = torch.ones_like(source_latent) + self.src_latents = source_latent.clone() + + logger.debug(f"[ACE15_NATIVE_EXTRACT] chunk_masks.shape: {self.chunk_masks.shape}") + + self._wrapper_applied = False + + def _apply_model_wrapper(self): + """Apply model wrapper to inject chunk_masks, src_latents, is_covers, and semantic hints""" + if self._wrapper_applied: + return + + chunk_masks = self.chunk_masks + src_latents = self.src_latents + semantic_hints = self.semantic_hints + + self.model_patcher.set_model_unet_function_wrapper(None) + + def model_function_wrapper(apply_model, args): + c = args["c"].copy() + device = args["input"].device + dtype = args["input"].dtype + + input_batch_size = args["input"].shape[0] + cm = chunk_masks.to(device=device, dtype=dtype) + sl = src_latents.to(device=device, dtype=dtype) + if cm.shape[0] < input_batch_size: + cm = cm.repeat(input_batch_size, 1, 1) + sl = sl.repeat(input_batch_size, 1, 1) + + c["chunk_masks"] = cm + c["src_latents"] = sl + + # Set is_covers based on whether we have semantic hints + if semantic_hints is not None: + sh = semantic_hints.to(device=device, dtype=dtype) + if sh.shape[0] < input_batch_size: + sh = sh.repeat(input_batch_size, 1, 1) + c["precomputed_lm_hints_25Hz"] = sh + c["is_covers"] = torch.ones((input_batch_size,), device=device, dtype=torch.long) + else: + c["is_covers"] = torch.zeros((input_batch_size,), device=device, dtype=torch.long) + + if not hasattr(model_function_wrapper, '_logged'): + logger.debug(f"[ACE15_EXTRACT_WRAPPER] Injecting extract parameters") + logger.debug(f"[ACE15_EXTRACT_WRAPPER] is_covers: {c['is_covers']}") + if "precomputed_lm_hints_25Hz" in c: + logger.debug(f"[ACE15_EXTRACT_WRAPPER] precomputed_lm_hints_25Hz.shape: {c['precomputed_lm_hints_25Hz'].shape}") + model_function_wrapper._logged = True + + return apply_model(args["input"], args["timestep"], **c) + + model_function_wrapper._logged = False + self.model_patcher.set_model_unet_function_wrapper(model_function_wrapper) + self._wrapper_applied = True + logger.debug(f"[ACE15_NATIVE_EXTRACT] Model wrapper applied") + + def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): + """Override sample to apply model wrapper for extract""" + + logger.debug(f"[ACE15_NATIVE_EXTRACT] sample() called") + + self._apply_model_wrapper() + + device = noise.device + dtype = noise.dtype + working_noise = torch.randn_like(self.source_latent, device=device, dtype=dtype) + + result = super().sample( + working_noise, + self.source_latent, + sampler, + sigmas, + denoise_mask, + callback, + disable_pbar, + seed + ) + + logger.debug(f"[ACE15_NATIVE_EXTRACT] result.shape: {result.shape}") + + return result + + +class ACEStep15NativeLegoGuider(comfy.samplers.CFGGuider): + """ + ACE-Step 1.5 lego guider for generating specific tracks within a region. + + Generates a specific track (e.g., vocals, drums) within a time region while + preserving the rest. Useful for adding instruments to existing audio. + + Official behavior: + - chunk_masks: 1s in generation region, 0s elsewhere + - src_latents: Original audio with generation region replaced by silence_latent + - Instruction: "Generate the {TRACK_NAME} track based on the audio context:" + + The silence_latent is automatically loaded internally (downloaded from HuggingFace + on first use). It's a learned tensor that tells the model to generate new content. + """ + + def __init__(self, model, positive, negative, cfg, source_latent, + track_name="vocals", start_seconds=0.0, end_seconds=None): + super().__init__(model) + self.set_conds(positive, negative) + self.set_cfg(cfg) + + self.source_latent = source_latent + self.track_name = track_name + + # Load silence_latent internally + from .ace_step_utils import load_silence_latent + silence_latent = load_silence_latent(verbose=True) + self.start_seconds = start_seconds + + # v1.5 frame rate: 48000 / 1920 = 25 fps + self.fps = 25.0 + + batch_size, channels, total_length = source_latent.shape + + # Calculate end time if not provided (use full duration) + if end_seconds is None: + end_seconds = total_length / self.fps + self.end_seconds = end_seconds + + self.start_frame = int(start_seconds * self.fps) + self.end_frame = int(end_seconds * self.fps) + + # Clamp frame indices to valid range + self.start_frame = max(0, min(self.start_frame, total_length)) + self.end_frame = max(0, min(self.end_frame, total_length)) + + logger.debug(f"[ACE15_NATIVE_LEGO] Initializing") + logger.debug(f"[ACE15_NATIVE_LEGO] source_latent.shape: {source_latent.shape}") + logger.debug(f"[ACE15_NATIVE_LEGO] silence_latent.shape: {silence_latent.shape}") + logger.debug(f"[ACE15_NATIVE_LEGO] track_name: {track_name}") + logger.debug(f"[ACE15_NATIVE_LEGO] region: {start_seconds}s-{end_seconds}s (frames {self.start_frame}-{self.end_frame})") + + # Lego: generate track in region, preserve elsewhere + self.chunk_masks = torch.zeros_like(source_latent) + self.chunk_masks[:, :, self.start_frame:self.end_frame] = 1.0 + + # src_latents: Original audio with generation region replaced by silence_latent + # Same logic as repaint - silence_latent shape is [1, T, D], need [batch, D, T] + silence_length = silence_latent.shape[1] # T dimension + lego_length = self.end_frame - self.start_frame + + # Tile silence_latent to at least total_length + silence_tiled = silence_latent[0, :total_length, :] # [total_length, D] or less + if silence_tiled.shape[0] < total_length: + # Need to tile/repeat + num_tiles = (total_length // silence_tiled.shape[0]) + 1 + silence_tiled = silence_latent[0].repeat(num_tiles, 1)[:total_length, :] + + # Transpose to [D, total_length] to match latent channel-first format + silence_tiled = silence_tiled.transpose(0, 1) # [D, total_length] + + # Create src_latents by replacing lego region with silence + self.src_latents = source_latent.clone() + # Move silence to same device/dtype as source + silence_region = silence_tiled[:, self.start_frame:self.end_frame].to( + device=source_latent.device, dtype=source_latent.dtype + ) + self.src_latents[:, :, self.start_frame:self.end_frame] = silence_region.unsqueeze(0).expand(batch_size, -1, -1) + + lego_frames = self.end_frame - self.start_frame + logger.debug(f"[ACE15_NATIVE_LEGO] chunk_masks.shape: {self.chunk_masks.shape}") + logger.debug(f"[ACE15_NATIVE_LEGO] lego frames: {lego_frames} ({lego_frames / self.fps:.2f}s)") + logger.debug(f"[ACE15_NATIVE_LEGO] src_latents lego region filled with silence_latent: frames {self.start_frame}-{self.end_frame}") + + self._wrapper_applied = False + + def _apply_model_wrapper(self): + """Apply model wrapper to inject chunk_masks and src_latents""" + if self._wrapper_applied: + return + + chunk_masks = self.chunk_masks + src_latents = self.src_latents + + self.model_patcher.set_model_unet_function_wrapper(None) + + def model_function_wrapper(apply_model, args): + c = args["c"].copy() + device = args["input"].device + dtype = args["input"].dtype + + input_batch_size = args["input"].shape[0] + cm = chunk_masks.to(device=device, dtype=dtype) + sl = src_latents.to(device=device, dtype=dtype) + if cm.shape[0] < input_batch_size: + cm = cm.repeat(input_batch_size, 1, 1) + sl = sl.repeat(input_batch_size, 1, 1) + + c["chunk_masks"] = cm + c["src_latents"] = sl + + if not hasattr(model_function_wrapper, '_logged'): + logger.debug(f"[ACE15_LEGO_WRAPPER] Injecting chunk_masks and src_latents") + model_function_wrapper._logged = True + + return apply_model(args["input"], args["timestep"], **c) + + model_function_wrapper._logged = False + self.model_patcher.set_model_unet_function_wrapper(model_function_wrapper) + self._wrapper_applied = True + logger.debug(f"[ACE15_NATIVE_LEGO] Model wrapper applied") + + def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): + """Override sample to apply model wrapper for lego""" + + logger.debug(f"[ACE15_NATIVE_LEGO] sample() called") + + self._apply_model_wrapper() + + device = noise.device + dtype = noise.dtype + working_noise = torch.randn_like(self.source_latent, device=device, dtype=dtype) + + result = super().sample( + working_noise, + self.source_latent, + sampler, + sigmas, + denoise_mask, + callback, + disable_pbar, + seed + ) + + logger.debug(f"[ACE15_NATIVE_LEGO] result.shape: {result.shape}") + + return result + + +# ============================================================================= +# ACE-Step 1.0 Guiders (use latent-level ODE blending) +# ============================================================================= + class ACEStepRepaintGuider(comfy.samplers.CFGGuider): """Repaint guider implementation""" - - def __init__(self, model, positive, negative, cfg, source_latent, + + def __init__(self, model, positive, negative, cfg, source_latent, start_time, end_time, repaint_strength, feather_time): super().__init__(model) self.set_conds(positive, negative) self.set_cfg(cfg) - + self.source_latent = source_latent self.start_time = start_time self.end_time = end_time self.repaint_strength = repaint_strength - + + # Detect version from latent shape + self.version = ACEStepLatentUtils.detect_version(source_latent) + self.is_v1_5 = self.version == ACEStepLatentUtils.V1_5 + # Convert times to frame indices - self.start_frame = ACEStepLatentUtils.time_to_frame_index(start_time) - self.end_frame = ACEStepLatentUtils.time_to_frame_index(end_time) - + self.start_frame = ACEStepLatentUtils.time_to_frame_index(start_time, self.version) + self.end_frame = ACEStepLatentUtils.time_to_frame_index(end_time, self.version) + # Create binary repaint mask (1.0 = generate new, 0.0 = preserve original) self.repaint_mask = torch.zeros_like(source_latent) - self.repaint_mask[:, :, :, self.start_frame:self.end_frame] = 1.0 - + if self.is_v1_5: + self.repaint_mask[:, :, self.start_frame:self.end_frame] = 1.0 + else: + self.repaint_mask[:, :, :, self.start_frame:self.end_frame] = 1.0 + # Apply feathering if requested if feather_time > 0: - feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time) + feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time, self.version) self.repaint_mask = ACEStepLatentUtils.create_feather_mask( self.repaint_mask, feather_frames) - + # Calculate n_min based on repaint_strength self.retake_variance = repaint_strength def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): """Override sample to implement repaint logic""" - + + logger.debug(f"[REPAINT_GUIDER] sample() called") + logger.debug(f"[REPAINT_GUIDER] noise.shape: {noise.shape}") + logger.debug(f"[REPAINT_GUIDER] latent_image.shape: {latent_image.shape}") + logger.debug(f"[REPAINT_GUIDER] self.is_v1_5: {self.is_v1_5}") + logger.debug(f"[REPAINT_GUIDER] self.version: {self.version}") + logger.debug(f"[REPAINT_GUIDER] self.source_latent.shape: {self.source_latent.shape}") + logger.debug(f"[REPAINT_GUIDER] self.repaint_mask.shape: {self.repaint_mask.shape}") + # Calculate n_min total_steps = len(sigmas) - 1 n_min = int(total_steps * (1 - self.retake_variance)) @@ -125,37 +818,45 @@ class ACEStepRepaintGuider(comfy.samplers.CFGGuider): class ACEStepExtendGuider(comfy.samplers.CFGGuider): """Extend guider implementation with custom ODE""" - + def __init__(self, model, positive, negative, cfg, source_latent, extend_left_time, extend_right_time): super().__init__(model) self.set_conds(positive, negative) self.set_cfg(cfg) - + self.source_latent = source_latent self.extend_left_time = extend_left_time self.extend_right_time = extend_right_time - - # Calculate frame extensions - self.left_frames = ACEStepLatentUtils.time_to_frame_index(extend_left_time) - self.right_frames = ACEStepLatentUtils.time_to_frame_index(extend_right_time) - + + # Detect version from latent shape + self.version = ACEStepLatentUtils.detect_version(source_latent) + self.is_v1_5 = self.version == ACEStepLatentUtils.V1_5 + + # Calculate frame extensions using version-specific FPS + self.left_frames = ACEStepLatentUtils.time_to_frame_index(extend_left_time, self.version) + self.right_frames = ACEStepLatentUtils.time_to_frame_index(extend_right_time, self.version) + # Store original source length for post-processing self.src_latents_length = source_latent.shape[-1] - - # Calculate max inference length (240 seconds = ~2640 frames) - self.max_infer_frame_length = int(240 * 44100 / 512 / 8) + + # Calculate max inference length (240 seconds) + fps = ACEStepLatentUtils.get_fps(self.version) + self.max_infer_frame_length = int(240 * fps) # Initialize trimming variables (matching community logic) self.left_trim_length = 0 self.right_trim_length = 0 self.to_left_pad_gt_latents = None self.to_right_pad_gt_latents = None - + # Create extended latent with trimming logic (matching community exactly) gt_latents = source_latent frame_length = self.left_frames + gt_latents.shape[-1] + self.right_frames - + + # Padding format differs: v1.5 is 3D (last dim), v1.0 is 4D (last dim) + # torch.nn.functional.pad pads last dimensions, so (left, right) works for both + # Handle left extension with trimming if self.left_frames > 0: extend_gt_latents = torch.nn.functional.pad( @@ -163,11 +864,15 @@ class ACEStepExtendGuider(comfy.samplers.CFGGuider): ) if frame_length > self.max_infer_frame_length: self.right_trim_length = frame_length - self.max_infer_frame_length - self.to_right_pad_gt_latents = extend_gt_latents[:, :, :, -self.right_trim_length:] - extend_gt_latents = extend_gt_latents[:, :, :, :self.max_infer_frame_length] + if self.is_v1_5: + self.to_right_pad_gt_latents = extend_gt_latents[:, :, -self.right_trim_length:] + extend_gt_latents = extend_gt_latents[:, :, :self.max_infer_frame_length] + else: + self.to_right_pad_gt_latents = extend_gt_latents[:, :, :, -self.right_trim_length:] + extend_gt_latents = extend_gt_latents[:, :, :, :self.max_infer_frame_length] frame_length = self.max_infer_frame_length gt_latents = extend_gt_latents - + # Handle right extension with trimming if self.right_frames > 0: extend_gt_latents = torch.nn.functional.pad( @@ -176,71 +881,118 @@ class ACEStepExtendGuider(comfy.samplers.CFGGuider): frame_length = extend_gt_latents.shape[-1] if frame_length > self.max_infer_frame_length: self.left_trim_length = frame_length - self.max_infer_frame_length - self.to_left_pad_gt_latents = extend_gt_latents[:, :, :, :self.left_trim_length] - extend_gt_latents = extend_gt_latents[:, :, :, -self.max_infer_frame_length:] + if self.is_v1_5: + self.to_left_pad_gt_latents = extend_gt_latents[:, :, :self.left_trim_length] + extend_gt_latents = extend_gt_latents[:, :, -self.max_infer_frame_length:] + else: + self.to_left_pad_gt_latents = extend_gt_latents[:, :, :, :self.left_trim_length] + extend_gt_latents = extend_gt_latents[:, :, :, -self.max_infer_frame_length:] frame_length = self.max_infer_frame_length gt_latents = extend_gt_latents - + # Store the working latent (this is what we'll use for inference) self.extended_latent = gt_latents - + # Create binary mask for extended regions self.extend_mask = torch.zeros_like(self.extended_latent) if self.left_frames > 0: actual_left_frames = min(self.left_frames, self.extended_latent.shape[-1]) - self.extend_mask[:, :, :, :actual_left_frames] = 1.0 + if self.is_v1_5: + self.extend_mask[:, :, :actual_left_frames] = 1.0 + else: + self.extend_mask[:, :, :, :actual_left_frames] = 1.0 if self.right_frames > 0: actual_right_frames = min(self.right_frames, self.extended_latent.shape[-1]) - self.extend_mask[:, :, :, -actual_right_frames:] = 1.0 - + if self.is_v1_5: + self.extend_mask[:, :, -actual_right_frames:] = 1.0 + else: + self.extend_mask[:, :, :, -actual_right_frames:] = 1.0 + # Calculate n_min based on retake_variance (using 1.0 for extend) self.retake_variance = 1.0 def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): """Override sample to implement extend logic exactly matching community implementation""" - + + logger.debug(f"[EXTEND_GUIDER] sample() called") + logger.debug(f"[EXTEND_GUIDER] noise.shape: {noise.shape}") + logger.debug(f"[EXTEND_GUIDER] latent_image.shape: {latent_image.shape}") + logger.debug(f"[EXTEND_GUIDER] self.is_v1_5: {self.is_v1_5}") + logger.debug(f"[EXTEND_GUIDER] self.version: {self.version}") + logger.debug(f"[EXTEND_GUIDER] self.extended_latent.shape: {self.extended_latent.shape}") + logger.debug(f"[EXTEND_GUIDER] self.source_latent.shape: {self.source_latent.shape}") + logger.debug(f"[EXTEND_GUIDER] self.left_frames: {self.left_frames}") + logger.debug(f"[EXTEND_GUIDER] self.right_frames: {self.right_frames}") + logger.debug(f"[EXTEND_GUIDER] self.left_trim_length: {self.left_trim_length}") + logger.debug(f"[EXTEND_GUIDER] self.right_trim_length: {self.right_trim_length}") + device = noise.device dtype = noise.dtype - + # Generate retake_latents - separate noise for extended regions (matches community) # Community uses randn_tensor with retake_random_generators, we approximate with randn_like retake_latents = torch.randn_like(self.extended_latent, device=device, dtype=dtype) - + logger.debug(f"[EXTEND_GUIDER] retake_latents.shape: {retake_latents.shape}") + # Generate target_latents for the FULL extended duration (this is what community does) # Community generates target_latents for the full frame_length, then slices it target_latents_full = torch.randn_like(self.extended_latent, device=device, dtype=dtype) - + logger.debug(f"[EXTEND_GUIDER] target_latents_full.shape: {target_latents_full.shape}") + # Create target_latents by concatenating pieces EXACTLY like community implementation # Community code lines 768-775: proper slicing with trimming padd_list = [] - + # Left extension: use retake_latents directly if self.left_frames > 0: actual_left_frames = min(self.left_frames, retake_latents.shape[-1]) - padd_list.append(retake_latents[:, :, :, :actual_left_frames]) - + logger.debug(f"[EXTEND_GUIDER] LEFT: actual_left_frames={actual_left_frames}") + # v1.5 is 3D, v1.0 is 4D - use appropriate indexing + if self.is_v1_5: + padd_list.append(retake_latents[:, :, :actual_left_frames]) + else: + padd_list.append(retake_latents[:, :, :, :actual_left_frames]) + # Middle section: use TRIMMED SLICE of target_latents_full (this is the key fix!) # Community: target_latents[left_trim_length : target_latents.shape[-1] - right_trim_length] middle_start = self.left_trim_length middle_end = target_latents_full.shape[-1] - self.right_trim_length - middle_slice = target_latents_full[:, :, :, middle_start:middle_end] - + logger.debug(f"[EXTEND_GUIDER] MIDDLE: middle_start={middle_start}, middle_end={middle_end}") + + # v1.5 is 3D, v1.0 is 4D - use appropriate indexing + if self.is_v1_5: + middle_slice = target_latents_full[:, :, middle_start:middle_end] + else: + middle_slice = target_latents_full[:, :, :, middle_start:middle_end] + logger.debug(f"[EXTEND_GUIDER] middle_slice.shape after slicing: {middle_slice.shape}") + # The middle slice should match the original source latent length # We need to trim it to match the source latent size if middle_slice.shape[-1] > self.src_latents_length: # Trim to source length - middle_slice = middle_slice[:, :, :, :self.src_latents_length] + logger.debug(f"[EXTEND_GUIDER] Trimming middle_slice from {middle_slice.shape[-1]} to {self.src_latents_length}") + if self.is_v1_5: + middle_slice = middle_slice[:, :, :self.src_latents_length] + else: + middle_slice = middle_slice[:, :, :, :self.src_latents_length] elif middle_slice.shape[-1] < self.src_latents_length: # This shouldn't happen in normal cases, but pad if needed padding_needed = self.src_latents_length - middle_slice.shape[-1] + logger.debug(f"[EXTEND_GUIDER] Padding middle_slice by {padding_needed}") middle_slice = torch.nn.functional.pad(middle_slice, (0, padding_needed), "constant", 0.0) - + + logger.debug(f"[EXTEND_GUIDER] middle_slice.shape final: {middle_slice.shape}") padd_list.append(middle_slice) - + # Right extension: use retake_latents directly if self.right_frames > 0: actual_right_frames = min(self.right_frames, retake_latents.shape[-1]) - padd_list.append(retake_latents[:, :, :, -actual_right_frames:]) + logger.debug(f"[EXTEND_GUIDER] RIGHT: actual_right_frames={actual_right_frames}") + # v1.5 is 3D, v1.0 is 4D - use appropriate indexing + if self.is_v1_5: + padd_list.append(retake_latents[:, :, -actual_right_frames:]) + else: + padd_list.append(retake_latents[:, :, :, -actual_right_frames:]) target_latents = torch.cat(padd_list, dim=-1) @@ -334,14 +1086,14 @@ class ACEStepExtendGuider(comfy.samplers.CFGGuider): class ACEStepHybridGuider(comfy.samplers.CFGGuider): """Simple guider that combines repaint and extend functionality using latent replacement""" - + def __init__(self, model, positive, negative, cfg, source_latent, repaint_start_time, repaint_end_time, repaint_strength, feather_time, extend_left_time, extend_right_time): super().__init__(model) self.set_conds(positive, negative) self.set_cfg(cfg) - + self.source_latent = source_latent self.repaint_start_time = repaint_start_time self.repaint_end_time = repaint_end_time @@ -349,32 +1101,37 @@ class ACEStepHybridGuider(comfy.samplers.CFGGuider): self.feather_time = feather_time self.extend_left_time = extend_left_time self.extend_right_time = extend_right_time - + + # Detect version from latent shape + self.version = ACEStepLatentUtils.detect_version(source_latent) + self.is_v1_5 = self.version == ACEStepLatentUtils.V1_5 + # Check which operations are enabled self.has_repaint = repaint_start_time is not None and repaint_end_time is not None self.has_extend = extend_left_time > 0 or extend_right_time > 0 - + # Setup extend if enabled (using improved logic from ACEStepExtendGuider) if self.has_extend: - self.left_frames = ACEStepLatentUtils.time_to_frame_index(extend_left_time) - self.right_frames = ACEStepLatentUtils.time_to_frame_index(extend_right_time) - + self.left_frames = ACEStepLatentUtils.time_to_frame_index(extend_left_time, self.version) + self.right_frames = ACEStepLatentUtils.time_to_frame_index(extend_right_time, self.version) + # Store original source length for post-processing self.src_latents_length = source_latent.shape[-1] - - # Calculate max inference length (240 seconds = ~2640 frames) - self.max_infer_frame_length = int(240 * 44100 / 512 / 8) + + # Calculate max inference length (240 seconds) + fps = ACEStepLatentUtils.get_fps(self.version) + self.max_infer_frame_length = int(240 * fps) # Initialize trimming variables (matching community logic) self.left_trim_length = 0 self.right_trim_length = 0 self.to_left_pad_gt_latents = None self.to_right_pad_gt_latents = None - + # Create extended latent with trimming logic (matching community exactly) gt_latents = source_latent frame_length = self.left_frames + gt_latents.shape[-1] + self.right_frames - + # Handle left extension with trimming if self.left_frames > 0: extend_gt_latents = torch.nn.functional.pad( @@ -382,11 +1139,15 @@ class ACEStepHybridGuider(comfy.samplers.CFGGuider): ) if frame_length > self.max_infer_frame_length: self.right_trim_length = frame_length - self.max_infer_frame_length - self.to_right_pad_gt_latents = extend_gt_latents[:, :, :, -self.right_trim_length:] - extend_gt_latents = extend_gt_latents[:, :, :, :self.max_infer_frame_length] + if self.is_v1_5: + self.to_right_pad_gt_latents = extend_gt_latents[:, :, -self.right_trim_length:] + extend_gt_latents = extend_gt_latents[:, :, :self.max_infer_frame_length] + else: + self.to_right_pad_gt_latents = extend_gt_latents[:, :, :, -self.right_trim_length:] + extend_gt_latents = extend_gt_latents[:, :, :, :self.max_infer_frame_length] frame_length = self.max_infer_frame_length gt_latents = extend_gt_latents - + # Handle right extension with trimming if self.right_frames > 0: extend_gt_latents = torch.nn.functional.pad( @@ -395,22 +1156,32 @@ class ACEStepHybridGuider(comfy.samplers.CFGGuider): frame_length = extend_gt_latents.shape[-1] if frame_length > self.max_infer_frame_length: self.left_trim_length = frame_length - self.max_infer_frame_length - self.to_left_pad_gt_latents = extend_gt_latents[:, :, :, :self.left_trim_length] - extend_gt_latents = extend_gt_latents[:, :, :, -self.max_infer_frame_length:] + if self.is_v1_5: + self.to_left_pad_gt_latents = extend_gt_latents[:, :, :self.left_trim_length] + extend_gt_latents = extend_gt_latents[:, :, -self.max_infer_frame_length:] + else: + self.to_left_pad_gt_latents = extend_gt_latents[:, :, :, :self.left_trim_length] + extend_gt_latents = extend_gt_latents[:, :, :, -self.max_infer_frame_length:] frame_length = self.max_infer_frame_length gt_latents = extend_gt_latents - + # Store the working latent (this is what we'll use for inference) self.extended_latent = gt_latents - + # Create binary mask for extended regions self.extend_mask = torch.zeros_like(self.extended_latent) if self.left_frames > 0: actual_left_frames = min(self.left_frames, self.extended_latent.shape[-1]) - self.extend_mask[:, :, :, :actual_left_frames] = 1.0 + if self.is_v1_5: + self.extend_mask[:, :, :actual_left_frames] = 1.0 + else: + self.extend_mask[:, :, :, :actual_left_frames] = 1.0 if self.right_frames > 0: actual_right_frames = min(self.right_frames, self.extended_latent.shape[-1]) - self.extend_mask[:, :, :, -actual_right_frames:] = 1.0 + if self.is_v1_5: + self.extend_mask[:, :, -actual_right_frames:] = 1.0 + else: + self.extend_mask[:, :, :, -actual_right_frames:] = 1.0 else: self.extended_latent = source_latent self.extend_mask = None @@ -424,17 +1195,20 @@ class ACEStepHybridGuider(comfy.samplers.CFGGuider): # Adjust repaint times for extended latent adjusted_start_time = repaint_start_time + extend_left_time adjusted_end_time = repaint_end_time + extend_left_time - - start_frame = ACEStepLatentUtils.time_to_frame_index(adjusted_start_time) - end_frame = ACEStepLatentUtils.time_to_frame_index(adjusted_end_time) - + + start_frame = ACEStepLatentUtils.time_to_frame_index(adjusted_start_time, self.version) + end_frame = ACEStepLatentUtils.time_to_frame_index(adjusted_end_time, self.version) + # Create repaint mask self.repaint_mask = torch.zeros_like(self.extended_latent) - self.repaint_mask[:, :, :, start_frame:end_frame] = 1.0 - + if self.is_v1_5: + self.repaint_mask[:, :, start_frame:end_frame] = 1.0 + else: + self.repaint_mask[:, :, :, start_frame:end_frame] = 1.0 + # Apply feathering if requested if feather_time > 0: - feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time) + feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time, self.version) self.repaint_mask = ACEStepLatentUtils.create_feather_mask( self.repaint_mask, feather_frames) else: @@ -449,10 +1223,18 @@ class ACEStepHybridGuider(comfy.samplers.CFGGuider): def sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None): """Override sample to handle hybrid repaint+extend logic""" - + + logger.debug(f"[HYBRID_GUIDER] sample() called") + logger.debug(f"[HYBRID_GUIDER] noise.shape: {noise.shape}") + logger.debug(f"[HYBRID_GUIDER] latent_image.shape: {latent_image.shape}") + logger.debug(f"[HYBRID_GUIDER] self.is_v1_5: {self.is_v1_5}") + logger.debug(f"[HYBRID_GUIDER] self.has_extend: {self.has_extend}") + logger.debug(f"[HYBRID_GUIDER] self.has_repaint: {self.has_repaint}") + # Use extended latent (which equals source latent if no extend) working_latent = self.extended_latent - + logger.debug(f"[HYBRID_GUIDER] working_latent.shape: {working_latent.shape}") + # Generate noise for working shape if self.has_extend: if hasattr(noise, 'seed'): @@ -461,43 +1243,59 @@ class ACEStepHybridGuider(comfy.samplers.CFGGuider): # Use improved noise generation strategy for better extend blending device = noise.device dtype = noise.dtype - + # Generate additional noise for extended regions retake_noise = torch.randn_like(self.extended_latent, device=device, dtype=dtype) - + # Generate target_latents for the FULL extended duration (matches community) target_latents_full = torch.randn_like(self.extended_latent, device=device, dtype=dtype) - + # Create noise by concatenating pieces EXACTLY like community implementation # Community does NOT use cosine/sine mixing for extend - only direct concatenation padd_list = [] - + # Left extension: use retake_noise directly if self.left_frames > 0: actual_left_frames = min(self.left_frames, retake_noise.shape[-1]) - padd_list.append(retake_noise[:, :, :, :actual_left_frames]) - + # v1.5 is 3D, v1.0 is 4D - use appropriate indexing + if self.is_v1_5: + padd_list.append(retake_noise[:, :, :actual_left_frames]) + else: + padd_list.append(retake_noise[:, :, :, :actual_left_frames]) + # Middle section: use TRIMMED SLICE of target_latents_full (key fix!) middle_start = self.left_trim_length middle_end = target_latents_full.shape[-1] - self.right_trim_length - middle_slice = target_latents_full[:, :, :, middle_start:middle_end] - + # v1.5 is 3D, v1.0 is 4D - use appropriate indexing + if self.is_v1_5: + middle_slice = target_latents_full[:, :, middle_start:middle_end] + else: + middle_slice = target_latents_full[:, :, :, middle_start:middle_end] + # Trim to source length if middle_slice.shape[-1] > self.src_latents_length: - middle_slice = middle_slice[:, :, :, :self.src_latents_length] + if self.is_v1_5: + middle_slice = middle_slice[:, :, :self.src_latents_length] + else: + middle_slice = middle_slice[:, :, :, :self.src_latents_length] elif middle_slice.shape[-1] < self.src_latents_length: padding_needed = self.src_latents_length - middle_slice.shape[-1] middle_slice = torch.nn.functional.pad(middle_slice, (0, padding_needed), "constant", 0.0) - + padd_list.append(middle_slice) - + # Right extension: use retake_noise directly if self.right_frames > 0: actual_right_frames = min(self.right_frames, retake_noise.shape[-1]) - padd_list.append(retake_noise[:, :, :, -actual_right_frames:]) - + # v1.5 is 3D, v1.0 is 4D - use appropriate indexing + if self.is_v1_5: + padd_list.append(retake_noise[:, :, -actual_right_frames:]) + else: + padd_list.append(retake_noise[:, :, :, -actual_right_frames:]) + working_noise = torch.cat(padd_list, dim=-1) - + logger.debug(f"[HYBRID_GUIDER] working_noise.shape: {working_noise.shape}") + # Ensure working_noise matches working_latent shape assert working_noise.shape[-1] == working_latent.shape[-1], \ f"Shape mismatch: {working_noise.shape=} vs {working_latent.shape=}" diff --git a/nodes/acestep/ace_step_utils.py b/nodes/acestep/ace_step_utils.py index d709a01..6ec2dac 100644 --- a/nodes/acestep/ace_step_utils.py +++ b/nodes/acestep/ace_step_utils.py @@ -1,116 +1,326 @@ import torch import torch.nn.functional as F import math +import os +import comfy.model_management +import folder_paths +from . import logger + +# Module-level cache for silence_latent (loaded once per session) +_silence_latent_cache = None + +# HuggingFace URL for silence_latent.pt +SILENCE_LATENT_URL = "https://huggingface.co/ACE-Step/Ace-Step1.5/resolve/main/acestep-v15-turbo/silence_latent.pt" + + +def get_silence_latent_path(): + """Get the path where silence_latent.pt should be stored.""" + models_dir = folder_paths.models_dir + ace_step_dir = os.path.join(models_dir, "ace_step") + return os.path.join(ace_step_dir, "silence_latent.pt") + + +def download_silence_latent(): + """Download silence_latent.pt from HuggingFace.""" + import urllib.request + + silence_latent_path = get_silence_latent_path() + ace_step_dir = os.path.dirname(silence_latent_path) + + # Create directory if it doesn't exist + os.makedirs(ace_step_dir, exist_ok=True) + + logger.info(f"[ACE-Step] Downloading silence_latent.pt from HuggingFace...") + logger.info(f"[ACE-Step] URL: {SILENCE_LATENT_URL}") + logger.info(f"[ACE-Step] Destination: {silence_latent_path}") + + try: + urllib.request.urlretrieve(SILENCE_LATENT_URL, silence_latent_path) + logger.info(f"[ACE-Step] Download complete!") + return True + except Exception as e: + logger.info(f"[ACE-Step] ERROR: Failed to download silence_latent.pt: {e}") + return False + + +def load_silence_latent(verbose=True): + """ + Load the silence_latent tensor, downloading if necessary. + + The silence_latent is a LEARNED tensor that represents "silence" in the latent space. + It is NOT zeros - it's a specific tensor that the model was trained with. + This tells the model to "generate new content here". + + Returns: + torch.Tensor: silence_latent in shape [1, T, D] (transposed from file) + """ + global _silence_latent_cache + + # Return cached version if available + if _silence_latent_cache is not None: + if verbose: + logger.info(f"[ACE-Step] Using cached silence_latent, shape: {_silence_latent_cache.shape}") + return _silence_latent_cache + + silence_latent_path = get_silence_latent_path() + + # Download if not exists + if not os.path.exists(silence_latent_path): + success = download_silence_latent() + if not success: + raise RuntimeError( + f"Failed to download silence_latent.pt. Please manually download from:\n" + f" {SILENCE_LATENT_URL}\n" + f"and place it at:\n" + f" {silence_latent_path}" + ) + + # Load the tensor + if verbose: + logger.info(f"[ACE-Step] Loading silence_latent from {silence_latent_path}") + silence_latent = torch.load(silence_latent_path, map_location="cpu", weights_only=True) + + # Transpose from [1, D, T] to [1, T, D] as per reference implementation + # handler.py line 465: self.silence_latent = torch.load(silence_latent_path).transpose(1, 2) + silence_latent = silence_latent.transpose(1, 2) + + if verbose: + logger.info(f"[ACE-Step] silence_latent loaded, shape: {silence_latent.shape}") + + # Cache for future use + _silence_latent_cache = silence_latent + + return silence_latent + + +def extract_semantic_hints(model, source_latent, verbose=True): + """ + Extract semantic hints from source audio latents for Cover/Extract tasks. + + This extracts the semantic structure of the audio (rhythm, melody, harmony) + while abstracting away the timbre. Used by Cover and Extract guiders. + + Args: + model: ComfyUI ModelPatcher wrapping the ACE-Step 1.5 model + source_latent: Source audio latent in ComfyUI format [B, 64, T] + verbose: Whether to print diagnostic information + + Returns: + torch.Tensor: Semantic hints in ComfyUI format [B, 64, T] + """ + prefix = "[SEMANTIC_EXTRACT]" + + # Validate v1.5 shape: (batch, 64, length) + if len(source_latent.shape) != 3 or source_latent.shape[1] != 64: + raise ValueError(f"ACE-Step 1.5 requires latent shape (batch, 64, length), got {source_latent.shape}") + + # Get the diffusion model (AceStepConditionGenerationModel) + diffusion_model = model.model.diffusion_model + + if not hasattr(diffusion_model, 'tokenizer') or not hasattr(diffusion_model, 'detokenizer'): + raise RuntimeError( + "Model does not have tokenizer/detokenizer. " + "Make sure you're using an ACE-Step 1.5 model." + ) + + # Load model to GPU + comfy.model_management.load_model_gpu(model) + + device = comfy.model_management.get_torch_device() + dtype = model.model.get_dtype() + + # Move source tensor to model device/dtype + # Source is in ComfyUI format: [B, D, T] = [B, 64, length] + # Tokenizer expects: [B, T, D] = [B, length, 64] + source_on_device = source_latent.to(device=device, dtype=dtype) + source_transposed = source_on_device.movedim(-1, -2) # [B, T, D] + + if verbose: + logger.debug(f"{prefix} Extracting from source shape: {source_latent.shape}") + logger.debug(f"{prefix} source stats: mean={source_transposed.mean():.4f}, std={source_transposed.std():.4f}") + + # Verify weights are on GPU + tokenizer_device = next(diffusion_model.tokenizer.parameters()).device + if verbose: + logger.debug(f"{prefix} tokenizer on: {tokenizer_device}, target: {device}") + + # Step 1: Tokenize - get quantized embeddings at 5Hz + with torch.no_grad(): + quantized, indices = diffusion_model.tokenizer.tokenize(source_transposed) + + if verbose: + logger.debug(f"{prefix} quantized shape: {quantized.shape}") + logger.debug(f"{prefix} quantized stats: mean={quantized.mean():.4f}, std={quantized.std():.4f}") + + # Step 2: Detokenize - upsample from 5Hz to 25Hz + # Use quantized directly (not get_output_from_indices) to avoid redundant computation + with torch.no_grad(): + lm_hints = diffusion_model.detokenizer(quantized) + + if verbose: + logger.debug(f"{prefix} lm_hints shape: {lm_hints.shape}") + logger.debug(f"{prefix} lm_hints stats: mean={lm_hints.mean():.4f}, std={lm_hints.std():.4f}") + if lm_hints.std() < 0.01: + logger.warning(f"{prefix} Very low variance in semantic hints!") + + # Transpose back to ComfyUI format: [B, T, D] → [B, D, T] + semantic_hints = lm_hints.movedim(-1, -2) + + return semantic_hints + class ACEStepLatentUtils: """Utility functions for ACEStep audio latent manipulation""" - + + # Version constants + V1_0 = "v1.0" + V1_5 = "v1.5" + + # Frame rates: frames per second + V1_0_FPS = 44100 / 512 / 8 # ≈ 10.77 fps + V1_5_FPS = 48000 / 1920 # = 25 fps + @staticmethod - def time_to_frame_index(time_seconds): - """Convert time in seconds to ACE latent frame index - ACE audio: 1 second = 44100 samples / 512 / 8 ≈ 10.8 frames + def detect_version(latent): + """Detect ACE-Step version from latent shape + v1.0: (batch, 8, 16, length) - 4D + v1.5: (batch, 64, length) - 3D """ - return int(time_seconds * 44100 / 512 / 8) - + if len(latent.shape) == 4 and latent.shape[1] == 8 and latent.shape[2] == 16: + return ACEStepLatentUtils.V1_0 + elif len(latent.shape) == 3 and latent.shape[1] == 64: + return ACEStepLatentUtils.V1_5 + else: + return None + @staticmethod - def frame_index_to_time(frame_index): + def get_fps(version): + """Get frames per second for a given version""" + if version == ACEStepLatentUtils.V1_5: + return ACEStepLatentUtils.V1_5_FPS + return ACEStepLatentUtils.V1_0_FPS + + @staticmethod + def time_to_frame_index(time_seconds, version=None): + """Convert time in seconds to ACE latent frame index + v1.0: 1 second = 44100 / 512 / 8 ≈ 10.77 frames + v1.5: 1 second = 48000 / 1920 = 25 frames + """ + fps = ACEStepLatentUtils.get_fps(version) + return int(time_seconds * fps) + + @staticmethod + def frame_index_to_time(frame_index, version=None): """Convert ACE latent frame index to time in seconds""" - return frame_index * 512 * 8 / 44100 - + fps = ACEStepLatentUtils.get_fps(version) + return frame_index / fps + + @staticmethod + def get_latent_length(latent): + """Get the time dimension length from latent (last dim for both versions)""" + return latent.shape[-1] + @staticmethod def validate_ace_latent_shape(latent): - """Validate that latent has correct ACE audio shape""" - if len(latent.shape) != 4: - raise ValueError(f"ACE latent must be 4D, got {len(latent.shape)}D") - - batch_size, channels, height, length = latent.shape - if channels != 8: - raise ValueError(f"ACE latent must have 8 channels, got {channels}") - if height != 16: - raise ValueError(f"ACE latent must have height 16, got {height}") - - return True + """Validate that latent has correct ACE audio shape (v1.0 or v1.5)""" + version = ACEStepLatentUtils.detect_version(latent) + if version is None: + if len(latent.shape) == 4: + raise ValueError(f"ACE v1.0 latent must have shape (batch, 8, 16, length), got {latent.shape}") + elif len(latent.shape) == 3: + raise ValueError(f"ACE v1.5 latent must have shape (batch, 64, length), got {latent.shape}") + else: + raise ValueError(f"ACE latent must be 3D (v1.5) or 4D (v1.0), got {len(latent.shape)}D") + return version @staticmethod - def create_repaint_mask(latent_shape, start_frame, end_frame): + def create_repaint_mask(latent_shape, start_frame, end_frame, version=None): """Create a mask for repainting a specific region - + Args: - latent_shape: (batch, 8, 16, length) + latent_shape: (batch, 8, 16, length) for v1.0 or (batch, 64, length) for v1.5 start_frame: Frame index to start repainting end_frame: Frame index to end repainting - + version: ACE-Step version (auto-detected if None) + Returns: torch.Tensor: Binary mask with 1.0 in repaint region """ - batch_size, channels, height, length = latent_shape - + length = latent_shape[-1] + # Clamp frame indices to valid range start_frame = max(0, start_frame) end_frame = min(length, end_frame) - + if start_frame >= end_frame: raise ValueError(f"start_frame ({start_frame}) must be less than end_frame ({end_frame})") - + mask = torch.zeros(latent_shape) - mask[:, :, :, start_frame:end_frame] = 1.0 - + if len(latent_shape) == 4: # v1.0 + mask[:, :, :, start_frame:end_frame] = 1.0 + else: # v1.5 + mask[:, :, start_frame:end_frame] = 1.0 + return mask - + @staticmethod def create_extend_mask(source_shape, left_frames, right_frames): """Create a mask for extension regions - + Args: - source_shape: (batch, 8, 16, length) shape of source latent + source_shape: (batch, 8, 16, length) for v1.0 or (batch, 64, length) for v1.5 left_frames: Number of frames to extend on the left right_frames: Number of frames to extend on the right - + Returns: torch.Tensor: Binary mask with 1.0 in extension regions """ - batch_size, channels, height, source_length = source_shape + source_length = source_shape[-1] total_length = source_length + left_frames + right_frames - - mask = torch.zeros((batch_size, channels, height, total_length)) - - # Mark left extension region - if left_frames > 0: - mask[:, :, :, :left_frames] = 1.0 - - # Mark right extension region - if right_frames > 0: - mask[:, :, :, -right_frames:] = 1.0 - + + if len(source_shape) == 4: # v1.0 + batch_size, channels, height, _ = source_shape + mask = torch.zeros((batch_size, channels, height, total_length)) + if left_frames > 0: + mask[:, :, :, :left_frames] = 1.0 + if right_frames > 0: + mask[:, :, :, -right_frames:] = 1.0 + else: # v1.5 + batch_size, channels, _ = source_shape + mask = torch.zeros((batch_size, channels, total_length)) + if left_frames > 0: + mask[:, :, :left_frames] = 1.0 + if right_frames > 0: + mask[:, :, -right_frames:] = 1.0 + return mask - + @staticmethod def create_extend_region_mask(extended_shape, left_frames, right_frames, source_length): """Create a mask indicating which regions to generate vs preserve in extended latent - + Args: - extended_shape: Shape of the extended latent (batch, 8, 16, total_length) + extended_shape: Shape of the extended latent left_frames: Number of frames added on the left right_frames: Number of frames added on the right source_length: Length of the original source latent - + Returns: torch.Tensor: Binary mask with 1.0 = generate (extension regions), 0.0 = preserve (source region) """ - batch_size, channels, height, total_length = extended_shape - mask = torch.zeros(extended_shape) - - # Mark left extension region for generation - if left_frames > 0: - mask[:, :, :, :left_frames] = 1.0 - - # Mark right extension region for generation - if right_frames > 0: - mask[:, :, :, -right_frames:] = 1.0 - - # The middle region (source_length) remains 0.0 for preservation - + + if len(extended_shape) == 4: # v1.0 + if left_frames > 0: + mask[:, :, :, :left_frames] = 1.0 + if right_frames > 0: + mask[:, :, :, -right_frames:] = 1.0 + else: # v1.5 + if left_frames > 0: + mask[:, :, :left_frames] = 1.0 + if right_frames > 0: + mask[:, :, -right_frames:] = 1.0 + return mask @staticmethod @@ -161,47 +371,52 @@ class ACEStepLatentUtils: @staticmethod def create_feather_mask(mask, feather_frames=2): """Create a feathered (soft) version of a binary mask - + Args: - mask: Binary mask tensor + mask: Binary mask tensor (3D for v1.5 or 4D for v1.0) feather_frames: Number of frames to feather on each side - + Returns: torch.Tensor: Feathered mask with smooth transitions """ if feather_frames <= 0: return mask - + # Create gaussian kernel for smoothing kernel_size = feather_frames * 2 + 1 sigma = feather_frames / 3.0 - + x = torch.arange(kernel_size, dtype=torch.float32) - feather_frames gaussian_kernel = torch.exp(-0.5 * (x / sigma) ** 2) gaussian_kernel = gaussian_kernel / gaussian_kernel.sum() - - # Reshape for 1D convolution along time dimension - kernel = gaussian_kernel.view(1, 1, 1, -1) - - # Apply convolution to each channel separately - batch_size, channels, height, length = mask.shape + result = torch.zeros_like(mask) - - for b in range(batch_size): - for c in range(channels): - for h in range(height): - # Extract 1D signal - signal = mask[b:b+1, c:c+1, h:h+1, :] # Shape: (1, 1, 1, length) - - # Pad with constant values (edge values) + is_v1_5 = len(mask.shape) == 3 + + if is_v1_5: + # v1.5: (batch, channels, length) + batch_size, channels, length = mask.shape + kernel = gaussian_kernel.view(1, 1, -1) + + for b in range(batch_size): + for c in range(channels): + signal = mask[b:b+1, c:c+1, :] padded_signal = F.pad(signal, (feather_frames, feather_frames), 'constant', 0.0) - - # Apply 1D convolution - smoothed = F.conv2d(padded_signal, kernel, padding=0) - - # Store result - result[b, c, h, :] = smoothed[0, 0, 0, :] - + smoothed = F.conv1d(padded_signal, kernel, padding=0) + result[b, c, :] = smoothed[0, 0, :] + else: + # v1.0: (batch, channels, height, length) + batch_size, channels, height, length = mask.shape + kernel = gaussian_kernel.view(1, 1, 1, -1) + + for b in range(batch_size): + for c in range(channels): + for h in range(height): + signal = mask[b:b+1, c:c+1, h:h+1, :] + padded_signal = F.pad(signal, (feather_frames, feather_frames), 'constant', 0.0) + smoothed = F.conv2d(padded_signal, kernel, padding=0) + result[b, c, h, :] = smoothed[0, 0, 0, :] + return torch.clamp(result, 0.0, 1.0) @staticmethod @@ -282,29 +497,29 @@ class ACEStepLatentUtils: class ACEStepRepaintHelper: """Helper class for repaint operations""" - - def __init__(self, source_latents, start_time, end_time, + + def __init__(self, source_latents, start_time, end_time, repaint_strength=0.7, feather_time=0.1): """ Args: - source_latents: Original audio latents (batch, 8, 16, length) + source_latents: Original audio latents (v1.0 or v1.5 shape) start_time: Start time in seconds for repaint region end_time: End time in seconds for repaint region repaint_strength: Strength of repainting (0.0 = no repaint, 1.0 = full repaint) feather_time: Time in seconds for feathering edges """ - ACEStepLatentUtils.validate_ace_latent_shape(source_latents) - + self.version = ACEStepLatentUtils.validate_ace_latent_shape(source_latents) + self.source_latents = source_latents - self.start_frame = ACEStepLatentUtils.time_to_frame_index(start_time) - self.end_frame = ACEStepLatentUtils.time_to_frame_index(end_time) + self.start_frame = ACEStepLatentUtils.time_to_frame_index(start_time, self.version) + self.end_frame = ACEStepLatentUtils.time_to_frame_index(end_time, self.version) self.repaint_strength = repaint_strength - self.feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time) - + self.feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time, self.version) + # Create masks self.repaint_mask = ACEStepLatentUtils.create_repaint_mask( - source_latents.shape, self.start_frame, self.end_frame) - + source_latents.shape, self.start_frame, self.end_frame, self.version) + # Apply feathering for smooth transitions if self.feather_frames > 0: self.repaint_mask = ACEStepLatentUtils.create_feather_mask( @@ -325,24 +540,24 @@ class ACEStepRepaintHelper: class ACEStepExtendHelper: """Helper class for extend operations""" - + def __init__(self, source_latents, extend_left_time=0, extend_right_time=0): """ Args: - source_latents: Original audio latents (batch, 8, 16, length) + source_latents: Original audio latents (v1.0 or v1.5 shape) extend_left_time: Time in seconds to extend before the audio extend_right_time: Time in seconds to extend after the audio """ - ACEStepLatentUtils.validate_ace_latent_shape(source_latents) - + self.version = ACEStepLatentUtils.validate_ace_latent_shape(source_latents) + self.source_latents = source_latents - self.left_frames = ACEStepLatentUtils.time_to_frame_index(extend_left_time) - self.right_frames = ACEStepLatentUtils.time_to_frame_index(extend_right_time) - + self.left_frames = ACEStepLatentUtils.time_to_frame_index(extend_left_time, self.version) + self.right_frames = ACEStepLatentUtils.time_to_frame_index(extend_right_time, self.version) + # Create extended latent with zero padding self.extended_latents = ACEStepLatentUtils.pad_latent_for_extend( source_latents, self.left_frames, self.right_frames) - + # Create extension mask self.extension_mask = ACEStepLatentUtils.create_extend_mask( source_latents.shape, self.left_frames, self.right_frames) diff --git a/nodes/acestep/audio_mask_nodes.py b/nodes/acestep/audio_mask_nodes.py index a41950a..a22ea7a 100644 --- a/nodes/acestep/audio_mask_nodes.py +++ b/nodes/acestep/audio_mask_nodes.py @@ -4,29 +4,40 @@ from .ace_step_utils import ACEStepLatentUtils class AudioMaskBase: """Base class for audio mask nodes with common functionality""" - + def _extract_and_validate_latent(self, audio_latents): - """Extract and validate ACE audio latent tensor""" + """Extract and validate ACE audio latent tensor, returns (tensor, version)""" if "samples" in audio_latents: latent_tensor = audio_latents["samples"] else: raise ValueError("Invalid latent format") - - ACEStepLatentUtils.validate_ace_latent_shape(latent_tensor) - return latent_tensor - - def _create_base_mask(self, latent_tensor): - """Create base mask with correct shape for CondPairSetProps""" - batch_size, channels, height, length = latent_tensor.shape - return torch.zeros((batch_size, height, length)), (batch_size, height, length) - + + version = ACEStepLatentUtils.validate_ace_latent_shape(latent_tensor) + return latent_tensor, version + + def _create_base_mask(self, latent_tensor, version=None): + """Create base mask with correct shape for conditioning + v1.0: (batch, height, length) where height=16 - matches noise.shape[2:] = (16, length) + Note: Temporal mask conditioning only works with ACE-Step 1.0 + """ + if version == ACEStepLatentUtils.V1_5: + # v1.5 uses discrete audio_codes tokens that cannot be interpolated + # Temporal mask conditioning is not supported for v1.5 + raise ValueError("Temporal mask conditioning is only supported for ACE-Step 1.0. " + "ACE-Step 1.5 uses discrete audio tokens that cannot be blended temporally.") + else: + batch_size, channels, height, length = latent_tensor.shape + mask = torch.zeros((batch_size, height, length)) + dims = (batch_size, height, length) + return mask, dims + def _finalize_mask(self, mask): """Ensure mask values are in valid range""" return torch.clamp(mask, 0.0, 1.0) class AudioTemporalMask(AudioMaskBase): """Create temporal crossfade mask from a list of float values""" - + @classmethod def INPUT_TYPES(s): return { @@ -36,16 +47,18 @@ class AudioTemporalMask(AudioMaskBase): "length_mismatch": (["repeat", "loop"], {"default": "repeat"}), } } - + RETURN_TYPES = ("MASK",) FUNCTION = "create_temporal_mask" CATEGORY = "conditioning/audio" - + def create_temporal_mask(self, audio_latents, value, length_mismatch): """Create temporal mask from custom float values""" - - latent_tensor = self._extract_and_validate_latent(audio_latents) - mask, (batch_size, height, length) = self._create_base_mask(latent_tensor) + + latent_tensor, version = self._extract_and_validate_latent(audio_latents) + mask, dims = self._create_base_mask(latent_tensor, version) + batch_size = dims[0] + length = dims[-1] # Always last dimension # Handle both single float and list of floats if isinstance(value, (int, float)): @@ -75,11 +88,14 @@ class AudioTemporalMask(AudioMaskBase): while len(values) < length: remaining = length - len(values) values.extend(values[:min(remaining, len(values))]) - - # Apply values to mask + + # Apply values to mask - v1.5 is 2D (batch, length), v1.0 is 3D (batch, height, length) for i in range(length): - mask[:, :, i] = values[i] - + if len(mask.shape) == 2: + mask[:, i] = values[i] + else: + mask[:, :, i] = values[i] + return (self._finalize_mask(mask),) class AudioRegionMask(AudioMaskBase): @@ -103,33 +119,44 @@ class AudioRegionMask(AudioMaskBase): def create_region_mask(self, audio_latents, start_time, end_time, mask_value, feather_seconds): """Create mask for specific time region with optional feathering""" + + latent_tensor, version = self._extract_and_validate_latent(audio_latents) + mask, dims = self._create_base_mask(latent_tensor, version) + length = dims[-1] # Always last dimension + + # Convert to frames using version-specific FPS + start_frame = ACEStepLatentUtils.time_to_frame_index(start_time, version) + end_frame = ACEStepLatentUtils.time_to_frame_index(end_time, version) + feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_seconds, version) - latent_tensor = self._extract_and_validate_latent(audio_latents) - mask, (batch_size, height, length) = self._create_base_mask(latent_tensor) - - # Convert to frames - start_frame = ACEStepLatentUtils.time_to_frame_index(start_time) - end_frame = ACEStepLatentUtils.time_to_frame_index(end_time) - feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_seconds) - - # Set region value + # Set region value - v1.5 is 2D (batch, length), v1.0 is 3D (batch, height, length) + is_2d = len(mask.shape) == 2 if start_frame < end_frame: - mask[:, :, start_frame:end_frame] = mask_value - + if is_2d: + mask[:, start_frame:end_frame] = mask_value + else: + mask[:, :, start_frame:end_frame] = mask_value + # Apply feathering if requested if feather_frames > 0: # Feather start for i in range(max(0, start_frame - feather_frames), start_frame): if i >= 0 and i < length: progress = (i - (start_frame - feather_frames)) / feather_frames - mask[:, :, i] = mask_value * progress - + if is_2d: + mask[:, i] = mask_value * progress + else: + mask[:, :, i] = mask_value * progress + # Feather end for i in range(end_frame, min(length, end_frame + feather_frames)): if i >= 0 and i < length: progress = 1.0 - (i - end_frame) / feather_frames - mask[:, :, i] = mask_value * progress - + if is_2d: + mask[:, i] = mask_value * progress + else: + mask[:, :, i] = mask_value * progress + return (self._finalize_mask(mask),) class AudioMaskAnalyzer: @@ -149,31 +176,44 @@ class AudioMaskAnalyzer: def analyze_mask(self, mask): """Show mask info and visualization""" - + info = f"Audio Mask Analysis:\n" info += f"Shape: {mask.shape}\n" - - if len(mask.shape) >= 3: - batch_size, height, length = mask.shape[0], mask.shape[1], mask.shape[-1] - duration = ACEStepLatentUtils.frame_index_to_time(length) - + + if len(mask.shape) >= 2: + batch_size = mask.shape[0] + length = mask.shape[-1] + + # Detect version based on shape - v1.5 is 2D, v1.0 is 3D + if len(mask.shape) == 2: + version = ACEStepLatentUtils.V1_5 + info += f"Version: v1.5 (2D mask)\n" + else: + version = ACEStepLatentUtils.V1_0 + info += f"Version: v1.0 (3D mask)\n" + + duration = ACEStepLatentUtils.frame_index_to_time(length, version) + info += f"Duration: {duration:.2f} seconds\n" info += f"Frames: {length}\n" - - # Get mask values along time dimension (first batch, first height) - mask_values = mask[0, 0, :].cpu().numpy() - + + # Get mask values along time dimension + if len(mask.shape) == 2: + mask_values = mask[0, :].cpu().numpy() + else: + mask_values = mask[0, 0, :].cpu().numpy() + info += f"Value range: {mask_values.min():.3f} to {mask_values.max():.3f}\n" - + # Create simple visualization viz_length = min(60, length) step = length / viz_length if viz_length > 0 else 1 - + viz_line = "" for i in range(viz_length): idx = int(i * step) val = mask_values[idx] if idx < len(mask_values) else 0 - + if val > 0.8: viz_line += "█" elif val > 0.6: @@ -184,11 +224,11 @@ class AudioMaskAnalyzer: viz_line += "░" else: viz_line += "·" - + info += f"\nTemporal pattern:\n{viz_line}\n" info += "█=0.8+ ▓=0.6+ ▒=0.4+ ░=0.2+ ·=<0.2\n" info += f"Time: 0s{'':>{viz_length-10}}{duration:.1f}s" - + return (info,) class MaskToAudioMask(AudioMaskBase): @@ -210,9 +250,11 @@ class MaskToAudioMask(AudioMaskBase): def convert_spatial_to_audio_mask(self, audio_latents, spatial_mask, frame_summary): """Convert spatial mask to audio temporal mask - batch=time, spatial→single value per frame""" - - latent_tensor = self._extract_and_validate_latent(audio_latents) - audio_mask, (batch_size, height, target_length) = self._create_base_mask(latent_tensor) + + latent_tensor, version = self._extract_and_validate_latent(audio_latents) + audio_mask, dims = self._create_base_mask(latent_tensor, version) + batch_size = dims[0] + target_length = dims[-1] # Handle different spatial mask shapes if len(spatial_mask.shape) == 2: @@ -261,11 +303,15 @@ class MaskToAudioMask(AudioMaskBase): else: final_pattern = torch.tensor(temporal_values, dtype=torch.float32) - # Apply temporal pattern to all batches and height channels + # Apply temporal pattern to all batches - v1.5 is 2D, v1.0 is 3D + is_2d = len(audio_mask.shape) == 2 for b in range(batch_size): - for h in range(height): - audio_mask[b, h, :] = final_pattern - + if is_2d: + audio_mask[b, :] = final_pattern + else: + for h in range(audio_mask.shape[1]): + audio_mask[b, h, :] = final_pattern + return (self._finalize_mask(audio_mask),) class AudioLatentInfo(AudioMaskBase): @@ -287,27 +333,32 @@ class AudioLatentInfo(AudioMaskBase): def get_audio_info(self, audio_latents, frame_rate): """Extract batch size, length, and duration from audio latents""" - - latent_tensor = self._extract_and_validate_latent(audio_latents) - batch_size, channels, height, actual_length = latent_tensor.shape - - # Calculate duration using the default frame rate (10.77) - default_frame_rate = 10.77 - duration = actual_length / default_frame_rate - + + latent_tensor, version = self._extract_and_validate_latent(audio_latents) + actual_length = latent_tensor.shape[-1] + batch_size = latent_tensor.shape[0] + channels = latent_tensor.shape[1] + + # Calculate duration using version-specific frame rate + native_fps = ACEStepLatentUtils.get_fps(version) + duration = actual_length / native_fps + # Calculate length_frames based on the specified frame rate length_frames = int(duration * frame_rate) - + # Create info string info = f"Audio Latent Info:\n" + info += f"Version: {version or 'unknown'}\n" info += f"Batch size: {batch_size}\n" info += f"Channels: {channels}\n" - info += f"Height: {height}\n" + if version != ACEStepLatentUtils.V1_5: + info += f"Height: {latent_tensor.shape[2]}\n" info += f"Actual length: {actual_length} frames\n" - info += f"Frame rate: {frame_rate:.2f} fps\n" - info += f"Length: {length_frames} frames\n" + info += f"Native frame rate: {native_fps:.2f} fps\n" + info += f"Target frame rate: {frame_rate:.2f} fps\n" + info += f"Length at target rate: {length_frames} frames\n" info += f"Duration: {duration:.2f} seconds" - + return (batch_size, length_frames, duration, info) # Node mappings @@ -320,9 +371,9 @@ AUDIO_MASK_NODE_CLASS_MAPPINGS = { } AUDIO_MASK_NODE_DISPLAY_NAME_MAPPINGS = { - "AudioTemporalMask": "Audio Temporal Mask", - "AudioRegionMask": "Audio Region Mask", + "AudioTemporalMask": "Audio Temporal Mask (ACE 1.0)", + "AudioRegionMask": "Audio Region Mask (ACE 1.0)", "AudioMaskAnalyzer": "Audio Mask Analyzer", - "MaskToAudioMask": "Mask to Audio Mask", + "MaskToAudioMask": "Mask to Audio Mask (ACE 1.0)", "AudioLatentInfo": "Audio Latent Info", } \ No newline at end of file diff --git a/nodes/acestep/nodes.py b/nodes/acestep/nodes.py index 4fe3898..8087cf4 100644 --- a/nodes/acestep/nodes.py +++ b/nodes/acestep/nodes.py @@ -1,7 +1,18 @@ import torch import comfy.model_management import comfy.samplers -from .ace_step_guiders import ACEStepRepaintGuider, ACEStepExtendGuider, ACEStepHybridGuider + +# Import patches module to apply ACE-Step 1.5 patches +from . import patches +from . import logger + +from .ace_step_guiders import ( + ACEStepRepaintGuider, ACEStepExtendGuider, ACEStepHybridGuider, + ACEStep15NativeEditGuider, + ACEStep15NativeCoverGuider, + # TODO: Re-enable when ready + # ACEStep15NativeExtractGuider, ACEStep15NativeLegoGuider +) from .ace_step_utils import ACEStepLatentUtils from .audio_mask_nodes import ( AudioTemporalMask, AudioRegionMask, AudioMaskAnalyzer, @@ -9,19 +20,123 @@ from .audio_mask_nodes import ( ) def validate_audio_latent(latents): - """Validate that latents are audio latents, handling both typed and untyped cases""" + """Validate that latents are audio latents, handling both typed and untyped cases + Supports both v1.0 (batch, 8, 16, length) and v1.5 (batch, 64, length) shapes + """ # Check if it's explicitly marked as audio type if latents.get("type") == "audio": return True - - # If no type field, check if it has the ACE audio latent shape (batch, 8, 16, length) + + # If no type field, check shape if "samples" in latents: tensor = latents["samples"] + # v1.0: (batch, 8, 16, length) if len(tensor.shape) == 4 and tensor.shape[1] == 8 and tensor.shape[2] == 16: return True - + # v1.5: (batch, 64, length) + if len(tensor.shape) == 3 and tensor.shape[1] == 64: + return True + return False + +class ACEStep15SemanticExtractor: + """ + Extracts semantic tokens from source audio latents for use with Cover/Extract tasks. + + The ACE-Step 1.5 model uses semantic tokens (lm_hints) as structural guidance + for tasks like Cover and Extract. This node extracts those tokens from VAE latents. + + Flow: VAE latents → tokenizer → quantized → detokenizer → semantic_hints + + Use the output with ACE-Step 1.5 Cover Guider or Extract Guider for creative + workflows like blending semantic hints from multiple sources. + + Note: Cover and Extract guiders will auto-extract semantic hints if not provided. + This node is useful for advanced workflows like semantic hint blending. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "source_latents": ("LATENT",), + } + } + + RETURN_TYPES = ("SEMANTIC_HINTS",) + RETURN_NAMES = ("semantic_hints",) + FUNCTION = "extract" + CATEGORY = "audio/acestep" + + def extract(self, model, source_latents): + from .ace_step_utils import extract_semantic_hints + + source_tensor = source_latents["samples"] + logger.debug(f"[ACE15_SEMANTIC_EXTRACTOR] Extracting semantic hints from shape {source_tensor.shape}") + + # Use shared utility function + semantic_hints = extract_semantic_hints(model, source_tensor, verbose=True) + + # Move to CPU for storage + semantic_hints = semantic_hints.cpu() + + return (semantic_hints,) + + +class ACEStep15SemanticHintsBlend: + """ + Blends semantic hints from two sources for creative mashups. + + Use this to: + - Create song mashups by blending structure from two sources + - Interpolate between musical structures + - Create hybrid compositions + + blend_factor: 0.0 = 100% hints_a, 1.0 = 100% hints_b, 0.5 = 50/50 mix + + TODO: Enhance with feature system for temporal blending, crossfades, + and region-based masking. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "hints_a": ("SEMANTIC_HINTS",), + "hints_b": ("SEMANTIC_HINTS",), + "blend_factor": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + } + } + + RETURN_TYPES = ("SEMANTIC_HINTS",) + RETURN_NAMES = ("blended_hints",) + FUNCTION = "blend" + CATEGORY = "audio/acestep" + + def blend(self, hints_a, hints_b, blend_factor): + # Validate shapes match + if hints_a.shape != hints_b.shape: + # Try to match lengths by truncating to shorter + min_length = min(hints_a.shape[-1], hints_b.shape[-1]) + hints_a = hints_a[..., :min_length] + hints_b = hints_b[..., :min_length] + logger.debug(f"[SEMANTIC_BLEND] Truncated to common length: {min_length}") + + logger.debug(f"[SEMANTIC_BLEND] Blending hints") + logger.debug(f"[SEMANTIC_BLEND] hints_a shape: {hints_a.shape}, stats: mean={hints_a.mean():.4f}, std={hints_a.std():.4f}") + logger.debug(f"[SEMANTIC_BLEND] hints_b shape: {hints_b.shape}, stats: mean={hints_b.mean():.4f}, std={hints_b.std():.4f}") + logger.debug(f"[SEMANTIC_BLEND] blend_factor: {blend_factor} (0=A, 1=B)") + + # Simple linear interpolation + blended = (1.0 - blend_factor) * hints_a + blend_factor * hints_b + + logger.debug(f"[SEMANTIC_BLEND] result stats: mean={blended.mean():.4f}, std={blended.std():.4f}") + + return (blended,) + + class ACEStepRepaintGuiderNode: """Node that creates a repaint guider for use with SamplerCustomAdvanced""" @@ -32,7 +147,7 @@ class ACEStepRepaintGuiderNode: "model": ("MODEL",), "positive": ("CONDITIONING",), "negative": ("CONDITIONING",), - "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), "source_latents": ("LATENT",), "start_time": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 1000.0, "step": 0.1}), "end_time": ("FLOAT", {"default": 20.0, "min": 0.0, "max": 1000.0, "step": 0.1}), @@ -77,7 +192,7 @@ class ACEStepExtendGuiderNode: "model": ("MODEL",), "positive": ("CONDITIONING",), "negative": ("CONDITIONING",), - "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), "source_latents": ("LATENT",), "extend_left_time": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step": 0.1}), "extend_right_time": ("FLOAT", {"default": 10.0, "min": 0.0, "max": 100.0, "step": 0.1}), @@ -90,22 +205,31 @@ class ACEStepExtendGuiderNode: def get_guider(self, model, positive, negative, cfg, source_latents, extend_left_time, extend_right_time): - + + logger.debug(f"[EXTEND_GUIDER_NODE] get_guider called") + logger.debug(f"[EXTEND_GUIDER_NODE] extend_left_time: {extend_left_time}") + logger.debug(f"[EXTEND_GUIDER_NODE] extend_right_time: {extend_right_time}") + if extend_left_time == 0 and extend_right_time == 0: raise ValueError("At least one of extend_left_time or extend_right_time must be > 0") - + if not validate_audio_latent(source_latents): raise ValueError("source_latents must be audio latents (from VAEEncodeAudio or EmptyAceStepLatentAudio)") - + # Extract latent tensor latent_tensor = source_latents["samples"] - + logger.debug(f"[EXTEND_GUIDER_NODE] latent_tensor.shape: {latent_tensor.shape}") + version = ACEStepLatentUtils.detect_version(latent_tensor) + logger.debug(f"[EXTEND_GUIDER_NODE] detected version: {version}") + # Create and return the extend guider guider = ACEStepExtendGuider( model, positive, negative, cfg, latent_tensor, extend_left_time, extend_right_time ) - + logger.debug(f"[EXTEND_GUIDER_NODE] guider.is_v1_5: {guider.is_v1_5}") + logger.debug(f"[EXTEND_GUIDER_NODE] guider.extended_latent.shape: {guider.extended_latent.shape}") + return (guider,) class ACEStepHybridGuiderNode: @@ -118,7 +242,7 @@ class ACEStepHybridGuiderNode: "model": ("MODEL",), "positive": ("CONDITIONING",), "negative": ("CONDITIONING",), - "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), "source_latents": ("LATENT",), "extend_left_time": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step": 0.1}), "extend_right_time": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step": 0.1}), @@ -186,24 +310,32 @@ class ACEStepAnalyzeLatentNode: def analyze_latent(self, latents): if not validate_audio_latent(latents): return ("Not audio latents", 0.0, 0, 0) - + latent_tensor = latents["samples"] - batch_size, channels, height, length = latent_tensor.shape - - # Calculate duration - duration = ACEStepLatentUtils.frame_index_to_time(length) - + version = ACEStepLatentUtils.detect_version(latent_tensor) + length = latent_tensor.shape[-1] + + # Calculate duration using version-specific FPS + duration = ACEStepLatentUtils.frame_index_to_time(length, version) + # Create info string info = f"ACE Audio Latent Analysis:\n" + info += f"Version: {version or 'unknown'}\n" info += f"Shape: {latent_tensor.shape}\n" - info += f"Batch size: {batch_size}\n" - info += f"Channels: {channels}\n" - info += f"Height: {height}\n" + info += f"Batch size: {latent_tensor.shape[0]}\n" + + if version == ACEStepLatentUtils.V1_5: + info += f"Channels: {latent_tensor.shape[1]}\n" + else: + info += f"Channels: {latent_tensor.shape[1]}\n" + info += f"Height: {latent_tensor.shape[2]}\n" + info += f"Length (frames): {length}\n" info += f"Duration: {duration:.2f} seconds\n" - info += f"Frame rate: ~{length/duration:.1f} frames/second" - - return (info, duration, length, channels) + fps = ACEStepLatentUtils.get_fps(version) + info += f"Frame rate: {fps:.2f} frames/second" + + return (info, duration, length, latent_tensor.shape[1]) class ACEStepTimeRangeNode: """Node for converting time ranges to frame indices for debugging""" @@ -258,24 +390,29 @@ class ACEStepMaskVisualizerNode: def visualize_mask(self, latents, start_time, end_time, feather_time): if not validate_audio_latent(latents): return ("Not audio latents",) - + latent_tensor = latents["samples"] - + version = ACEStepLatentUtils.detect_version(latent_tensor) + # Create mask - start_frame = ACEStepLatentUtils.time_to_frame_index(start_time) - end_frame = ACEStepLatentUtils.time_to_frame_index(end_time) - feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time) - - mask = ACEStepLatentUtils.create_repaint_mask(latent_tensor.shape, start_frame, end_frame) - + start_frame = ACEStepLatentUtils.time_to_frame_index(start_time, version) + end_frame = ACEStepLatentUtils.time_to_frame_index(end_time, version) + feather_frames = ACEStepLatentUtils.time_to_frame_index(feather_time, version) + + mask = ACEStepLatentUtils.create_repaint_mask(latent_tensor.shape, start_frame, end_frame, version) + if feather_frames > 0: feathered_mask = ACEStepLatentUtils.create_feather_mask(mask, feather_frames) else: feathered_mask = mask - + # Create visualization total_frames = latent_tensor.shape[-1] - mask_summary = feathered_mask[0, 0, 0, :].cpu().numpy() # Take first channel of first batch + # Handle both 3D (v1.5) and 4D (v1.0) shapes + if len(feathered_mask.shape) == 3: + mask_summary = feathered_mask[0, 0, :].cpu().numpy() + else: + mask_summary = feathered_mask[0, 0, 0, :].cpu().numpy() info = f"Mask Visualization (Length: {total_frames} frames):\n" info += f"Repaint region: Frame {start_frame} to {end_frame}\n" @@ -361,7 +498,11 @@ class ACEStepAudioPostProcessor: # Check if this is an extend guider to determine padding strategy if hasattr(guider, 'left_frames') and hasattr(guider, 'right_frames'): # We know the extend amounts, pad accordingly - left_samples = int(guider.left_frames * 512 * 8 / 44100 * orig_sr) + # Use version-aware conversion if available + if hasattr(guider, 'version') and guider.version == ACEStepLatentUtils.V1_5: + left_samples = int(guider.left_frames / 25.0 * orig_sr) # v1.5: 25 fps + else: + left_samples = int(guider.left_frames * 512 * 8 / 44100 * orig_sr) # v1.0 right_samples = proc_length - orig_length - left_samples @@ -431,10 +572,14 @@ class ACEStepAudioPostProcessor: # Latent frames correspond to audio segments latent_frames = latent_mask.shape[-1] audio_frames_per_latent = audio_length // latent_frames - + # Upsample mask to audio resolution # Take the first channel/batch element for the mask pattern - mask_pattern = latent_mask[0, 0, 0, :].cpu() # Shape: [latent_frames] + # Handle both v1.5 (3D) and v1.0 (4D) shapes + if len(latent_mask.shape) == 3: + mask_pattern = latent_mask[0, 0, :].cpu() # v1.5: Shape: [latent_frames] + else: + mask_pattern = latent_mask[0, 0, 0, :].cpu() # v1.0: Shape: [latent_frames] # Repeat each latent frame mask value for corresponding audio samples audio_mask = mask_pattern.repeat_interleave(audio_frames_per_latent) @@ -497,26 +642,426 @@ class ACEStepAudioPostProcessor: return smoothed_mask + +# ============================================================================= +# ACE-Step 1.5 Native Guider Nodes +# These use the model's native mask input via model wrapping +# ============================================================================= + +class ACEStep15NativeEditGuiderNode: + """ + Unified ACE-Step 1.5 edit guider for extend and/or repaint operations. + + This guider injects chunk_masks and src_latents directly into the model's forward() + method using model wrapping, rather than manipulating latents during sampling. + + Operations: + - Extend: Set extend_left_seconds and/or extend_right_seconds > 0 + - Repaint: Set repaint_start_seconds and repaint_end_seconds (repaint_end > repaint_start) + - Both: Combine extend and repaint in a single operation + + The silence_latent is automatically loaded internally (downloaded from HuggingFace + on first use). It's a learned tensor that tells the model to generate new content. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "source_latents": ("LATENT",), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + }, + "optional": { + "extend_left_seconds": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 240.0, "step": 0.1}), + "extend_right_seconds": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 240.0, "step": 0.1}), + "repaint_start_seconds": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1000.0, "step": 0.1}), + "repaint_end_seconds": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1000.0, "step": 0.1}), + } + } + + RETURN_TYPES = ("GUIDER", "LATENT") + RETURN_NAMES = ("guider", "output_latent") + FUNCTION = "get_guider" + CATEGORY = "sampling/custom_sampling/guiders" + + def get_guider(self, model, positive, negative, source_latents, cfg, + extend_left_seconds=0.0, extend_right_seconds=0.0, + repaint_start_seconds=-1.0, repaint_end_seconds=-1.0): + + source_tensor = source_latents["samples"] + + # Validate v1.5 shape: (batch, 64, length) + if len(source_tensor.shape) != 3 or source_tensor.shape[1] != 64: + raise ValueError(f"ACE-Step 1.5 requires latent shape (batch, 64, length), got {source_tensor.shape}. Use the v1.0 guiders for v1.0 latents.") + + # Determine which operations are enabled + has_extend = extend_left_seconds > 0 or extend_right_seconds > 0 + has_repaint = repaint_start_seconds >= 0 and repaint_end_seconds > repaint_start_seconds + + if not has_extend and not has_repaint: + raise ValueError("At least one operation must be enabled: set extend_left/right_seconds > 0 or set valid repaint_start/end_seconds") + + # Convert -1 values to None for the guider + repaint_start = repaint_start_seconds if repaint_start_seconds >= 0 else None + repaint_end = repaint_end_seconds if repaint_end_seconds >= 0 else None + + logger.debug(f"[ACE15_EDIT_NODE] Creating guider") + logger.debug(f"[ACE15_EDIT_NODE] source_tensor.shape: {source_tensor.shape}") + if has_extend: + logger.debug(f"[ACE15_EDIT_NODE] extend: left={extend_left_seconds}s, right={extend_right_seconds}s") + if has_repaint: + logger.debug(f"[ACE15_EDIT_NODE] repaint: {repaint_start_seconds}s - {repaint_end_seconds}s") + + # Create the unified guider (silence_latent is loaded internally) + guider = ACEStep15NativeEditGuider( + model, positive, negative, cfg, + source_tensor, + extend_left_seconds=extend_left_seconds, + extend_right_seconds=extend_right_seconds, + repaint_start_seconds=repaint_start, + repaint_end_seconds=repaint_end + ) + + # Return the working latent (may be extended or same as source) + output_latent = {"samples": guider.working_latent, "type": "audio"} + + return (guider, output_latent) + + +class ACEStep15NativeCoverGuiderNode: + """ + Creates an ACE-Step 1.5 cover guider for style transfer/regeneration. + + Uses semantic tokens from the source audio as structural guidance while + generating new content with a different style/timbre. + + Semantic hints are automatically extracted if not provided. For advanced + workflows like blending hints from multiple sources, use the Semantic + Extractor and Semantic Hints Blend nodes. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "source_latents": ("LATENT",), + }, + "optional": { + "semantic_hints": ("SEMANTIC_HINTS",), + } + } + + RETURN_TYPES = ("GUIDER",) + FUNCTION = "get_guider" + CATEGORY = "sampling/custom_sampling/guiders" + + def get_guider(self, model, positive, negative, cfg, source_latents, semantic_hints=None): + source_tensor = source_latents["samples"] + + # Validate v1.5 shape: (batch, 64, length) + if len(source_tensor.shape) != 3 or source_tensor.shape[1] != 64: + raise ValueError(f"ACE-Step 1.5 requires latent shape (batch, 64, length), got {source_tensor.shape}") + + logger.debug(f"[ACE15_COVER_NODE] Creating guider") + logger.debug(f"[ACE15_COVER_NODE] source_tensor.shape: {source_tensor.shape}") + if semantic_hints is not None: + logger.debug(f"[ACE15_COVER_NODE] semantic_hints provided externally: {semantic_hints.shape}") + else: + logger.debug(f"[ACE15_COVER_NODE] semantic_hints will be auto-extracted") + + guider = ACEStep15NativeCoverGuider( + model, positive, negative, cfg, source_tensor, semantic_hints + ) + + return (guider,) + + +class ACEStep15NativeExtractGuiderNode: + """ + Creates an ACE-Step 1.5 extract guider for extracting specific tracks. + + Extracts a specific track (vocals, drums, bass, etc.) from the source audio. + Use with the ACEStep15TaskTextEncode node with task_type="extract". + + Semantic hints are automatically extracted if not provided. For advanced + workflows, use the Semantic Extractor node. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "source_latents": ("LATENT",), + "track_name": (["vocals", "drums", "bass", "guitar", "keyboard", "strings", + "percussion", "synth", "fx", "brass", "woodwinds", "backing_vocals"],), + }, + "optional": { + "semantic_hints": ("SEMANTIC_HINTS",), + } + } + + RETURN_TYPES = ("GUIDER",) + FUNCTION = "get_guider" + CATEGORY = "sampling/custom_sampling/guiders" + + def get_guider(self, model, positive, negative, cfg, source_latents, track_name, semantic_hints=None): + source_tensor = source_latents["samples"] + + # Validate v1.5 shape: (batch, 64, length) + if len(source_tensor.shape) != 3 or source_tensor.shape[1] != 64: + raise ValueError(f"ACE-Step 1.5 requires latent shape (batch, 64, length), got {source_tensor.shape}") + + logger.debug(f"[ACE15_EXTRACT_NODE] Creating guider") + logger.debug(f"[ACE15_EXTRACT_NODE] source_tensor.shape: {source_tensor.shape}") + logger.debug(f"[ACE15_EXTRACT_NODE] track_name: {track_name}") + if semantic_hints is not None: + logger.debug(f"[ACE15_EXTRACT_NODE] semantic_hints provided externally: {semantic_hints.shape}") + else: + logger.debug(f"[ACE15_EXTRACT_NODE] semantic_hints will be auto-extracted") + + guider = ACEStep15NativeExtractGuider( + model, positive, negative, cfg, source_tensor, track_name, semantic_hints + ) + + return (guider,) + + +class ACEStep15NativeLegoGuiderNode: + """ + Creates an ACE-Step 1.5 lego guider for generating specific tracks in a region. + + Generates a specific track (vocals, drums, etc.) within a time region while + preserving the rest. Useful for adding instruments to existing audio. + Use with the ACEStep15TaskTextEncode node with task_type="lego". + + The silence_latent is automatically loaded internally (downloaded from HuggingFace + on first use). It's a learned tensor that tells the model to generate new content. + """ + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ("MODEL",), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "cfg": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "source_latents": ("LATENT",), + "track_name": (["vocals", "drums", "bass", "guitar", "keyboard", "strings", + "percussion", "synth", "fx", "brass", "woodwinds", "backing_vocals"],), + "start_seconds": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1000.0, "step": 0.1}), + "end_seconds": ("FLOAT", {"default": 30.0, "min": 0.0, "max": 1000.0, "step": 0.1}), + } + } + + RETURN_TYPES = ("GUIDER",) + FUNCTION = "get_guider" + CATEGORY = "sampling/custom_sampling/guiders" + + def get_guider(self, model, positive, negative, cfg, source_latents, + track_name, start_seconds, end_seconds): + if start_seconds >= end_seconds: + raise ValueError(f"start_seconds ({start_seconds}) must be less than end_seconds ({end_seconds})") + + source_tensor = source_latents["samples"] + + # Validate v1.5 shape: (batch, 64, length) + if len(source_tensor.shape) != 3 or source_tensor.shape[1] != 64: + raise ValueError(f"ACE-Step 1.5 requires latent shape (batch, 64, length), got {source_tensor.shape}") + + logger.debug(f"[ACE15_LEGO_NODE] Creating guider") + logger.debug(f"[ACE15_LEGO_NODE] source_tensor.shape: {source_tensor.shape}") + logger.debug(f"[ACE15_LEGO_NODE] track_name: {track_name}") + logger.debug(f"[ACE15_LEGO_NODE] region: {start_seconds}s - {end_seconds}s") + + # Create guider (silence_latent is loaded internally) + guider = ACEStep15NativeLegoGuider( + model, positive, negative, cfg, source_tensor, + track_name, start_seconds, end_seconds + ) + + return (guider,) + + +class ACEStep15TaskTextEncodeNode: + """ + Task-aware text encoder for ACE-Step 1.5. + + Encodes text with task-specific instructions: + - text2music: Generate audio from text description + - repaint: Regenerate a region while preserving the rest + - cover: Style transfer/regeneration with source audio as context + - extract: Extract a specific track (requires track_name) + - lego: Generate a specific track in a region (requires track_name) + + The task_type determines the instruction prefix used by the model. + For extract/lego tasks, track_name specifies which track to extract/generate. + + Returns single conditioning output. Use ConditioningZeroOut node for negative + conditioning (standard ACE-Step workflow pattern). + """ + + # Valid keyscales from reference: 7 notes × 3 accidentals (plain, #, b) × 2 modes = 42 combinations + # Using ASCII accidentals for compatibility + KEYSCALE_NOTES = ['C', 'D', 'E', 'F', 'G', 'A', 'B'] + KEYSCALE_ACCIDENTALS = ['', '#', 'b'] + KEYSCALE_MODES = ['major', 'minor'] + + # Generate all valid keyscales + VALID_KEYSCALES = [] + for note in KEYSCALE_NOTES: + for acc in KEYSCALE_ACCIDENTALS: + for mode in KEYSCALE_MODES: + VALID_KEYSCALES.append(f"{note}{acc} {mode}") + + # Valid time signatures from reference + VALID_TIME_SIGNATURES = ["2", "3", "4", "6"] + + # Valid languages from reference + VALID_LANGUAGES = [ + 'en', 'zh', 'ja', 'ko', 'es', 'fr', 'de', 'it', 'pt', 'ru', + 'ar', 'hi', 'vi', 'th', 'id', 'ms', 'tl', 'nl', 'pl', 'tr', + 'sv', 'da', 'no', 'fi', 'cs', 'sk', 'hu', 'ro', 'bg', 'hr', + 'sr', 'uk', 'el', 'he', 'fa', 'bn', 'ta', 'te', 'pa', 'ur', + 'ne', 'sw', 'ht', 'is', 'lt', 'la', 'az', 'ca', 'sa', 'yue', + 'unknown' + ] + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "clip": ("CLIP",), + "text": ("STRING", {"multiline": True, "dynamicPrompts": True, "default": "A melodic electronic track with soft synths"}), + "task_type": (["text2music", "repaint", "cover"],), # TODO: Re-enable "extract", "lego" when ready + }, + "optional": { + "track_name": (["", "vocals", "drums", "bass", "guitar", "keyboard", "strings", + "percussion", "synth", "fx", "brass", "woodwinds", "backing_vocals"], + {"default": ""}), + "lyrics": ("STRING", {"multiline": True, "default": ""}), + "bpm": ("INT", {"default": 120, "min": 30, "max": 300}), + "duration": ("INT", {"default": 60, "min": 10, "max": 600}), + # Use COMBO type to accept connections from AudioInfo detected_key output + "keyscale": ("COMBO", {"default": "C major", "options": s.VALID_KEYSCALES}), + "timesignature": ("COMBO", {"default": "4", "options": s.VALID_TIME_SIGNATURES}), + "language": ("COMBO", {"default": "en", "options": s.VALID_LANGUAGES}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffff}), + } + } + + RETURN_TYPES = ("CONDITIONING",) + RETURN_NAMES = ("conditioning",) + FUNCTION = "encode" + CATEGORY = "conditioning" + + def encode(self, clip, text, task_type, track_name="", lyrics="", bpm=120, + duration=60, keyscale="C major", timesignature="4", language="en", seed=0): + # Validate track_name for extract/lego tasks + if task_type in ["extract", "lego"] and not track_name: + logger.debug(f"[ACE15_TEXT_ENCODE] Warning: track_name not specified for {task_type} task, using default instruction") + + # Convert timesignature from string to int + timesig_int = int(timesignature) + + # Build kwargs for tokenizer + kwargs = { + "lyrics": lyrics, + "bpm": bpm, + "duration": duration, + "keyscale": keyscale, + "timesignature": timesig_int, + "language": language, + "seed": seed, + "task_type": task_type, + "track_name": track_name if track_name else None, + } + + logger.debug(f"[ACE15_TEXT_ENCODE] Encoding with task_type={task_type}, track_name={track_name or 'N/A'}") + + # Import to get task instruction for verification + from .patches import get_task_instruction, is_patched + instruction = get_task_instruction(task_type, track_name if track_name else None) + logger.debug(f"[ACE15_TEXT_ENCODE] patches applied: {is_patched()}") + logger.debug(f"[ACE15_TEXT_ENCODE] task instruction: {instruction}") + + # Use the patched tokenize_with_weights which accepts task_type + # encode_from_tokens_scheduled returns conditioning directly (list format) + tokens = clip.tokenize(text, **kwargs) + + # Verify the tokens contain task_type info (patched tokenizer should include this) + if isinstance(tokens, dict): + if "task_type" in tokens: + logger.debug(f"[ACE15_TEXT_ENCODE] tokens contain task_type: {tokens.get('task_type')}") + else: + logger.debug(f"[ACE15_TEXT_ENCODE] WARNING: tokens dict does not contain task_type - patch may not be applied!") + logger.debug(f"[ACE15_TEXT_ENCODE] tokens keys: {list(tokens.keys())}") + + conditioning = clip.encode_from_tokens_scheduled(tokens) + + # For negative conditioning, use ComfyUI's ConditioningZeroOut node + # This matches the standard ACE-Step workflow pattern + + return (conditioning,) + + # Node class mappings for ComfyUI NODE_CLASS_MAPPINGS = { + # ACE-Step 1.0 guiders (latent-level ODE blending) "ACEStepRepaintGuider": ACEStepRepaintGuiderNode, "ACEStepExtendGuider": ACEStepExtendGuiderNode, "ACEStepHybridGuider": ACEStepHybridGuiderNode, + # Utility nodes "ACEStepAnalyzeLatent": ACEStepAnalyzeLatentNode, "ACEStepTimeRange": ACEStepTimeRangeNode, "ACEStepMaskVisualizer": ACEStepMaskVisualizerNode, "ACEStepAudioPostProcessor": ACEStepAudioPostProcessor, + # ACE-Step 1.5 semantic extraction and blending + "ACEStep15SemanticExtractor": ACEStep15SemanticExtractor, + "ACEStep15SemanticHintsBlend": ACEStep15SemanticHintsBlend, + # ACE-Step 1.5 guiders (model-level mask input) + "ACEStep15NativeEditGuider": ACEStep15NativeEditGuiderNode, + "ACEStep15NativeCoverGuider": ACEStep15NativeCoverGuiderNode, + # TODO: Re-enable when ready + # "ACEStep15NativeExtractGuider": ACEStep15NativeExtractGuiderNode, + # "ACEStep15NativeLegoGuider": ACEStep15NativeLegoGuiderNode, + # ACE-Step 1.5 text encoder + "ACEStep15TaskTextEncode": ACEStep15TaskTextEncodeNode, **AUDIO_MASK_NODE_CLASS_MAPPINGS, # Add audio mask nodes } # Display name mappings for ComfyUI NODE_DISPLAY_NAME_MAPPINGS = { - "ACEStepRepaintGuider": "ACEStep Repaint Guider BETA", - "ACEStepExtendGuider": "ACEStep Extend Guider BETA", - "ACEStepHybridGuider": "ACEStep Hybrid Guider BETA", - "ACEStepAnalyzeLatent": "ACEStep Analyze Latent BETA", - "ACEStepTimeRange": "ACEStep Time Range BETA", - "ACEStepMaskVisualizer": "ACEStep Mask Visualizer BETA", - "ACEStepAudioPostProcessor": "ACEStep Audio Post Processor BETA", - **AUDIO_MASK_NODE_DISPLAY_NAME_MAPPINGS, # Add audio mask node display names + # ACE-Step 1.0 guiders + "ACEStepRepaintGuider": "ACEStep 1.0 Repaint Guider BETA", + "ACEStepExtendGuider": "ACEStep 1.0 Extend Guider BETA", + "ACEStepHybridGuider": "ACEStep 1.0 Hybrid Guider BETA", + # Utility nodes + "ACEStepAnalyzeLatent": "ACEStep Analyze Latent", + "ACEStepTimeRange": "ACEStep Time Range", + "ACEStepMaskVisualizer": "ACEStep Mask Visualizer", + "ACEStepAudioPostProcessor": "ACEStep Audio Post Processor", + # ACE-Step 1.5 semantic extraction and blending + "ACEStep15SemanticExtractor": "ACE-Step 1.5 Semantic Extractor", + "ACEStep15SemanticHintsBlend": "ACE-Step 1.5 Semantic Hints Blend", + # ACE-Step 1.5 guiders + "ACEStep15NativeEditGuider": "ACE-Step 1.5 Edit Guider (Extend/Repaint)", + "ACEStep15NativeCoverGuider": "ACE-Step 1.5 Cover Guider", + # TODO: Re-enable when ready + # "ACEStep15NativeExtractGuider": "ACE-Step 1.5 Extract Guider", + # "ACEStep15NativeLegoGuider": "ACE-Step 1.5 Lego Guider", + # ACE-Step 1.5 text encoder + "ACEStep15TaskTextEncode": "ACE-Step 1.5 Task Text Encode", + **AUDIO_MASK_NODE_DISPLAY_NAME_MAPPINGS, } \ No newline at end of file diff --git a/nodes/acestep/patches.py b/nodes/acestep/patches.py new file mode 100644 index 0000000..cee7f36 --- /dev/null +++ b/nodes/acestep/patches.py @@ -0,0 +1,260 @@ +""" +Monkey patches for ACE-Step 1.5 support in ComfyUI. + +These patches extend ComfyUI's ACE-Step implementation to support all task types: +- text2music: Pure generation from text +- repaint: Regenerate a region while preserving the rest +- extend: Add new audio before/after existing audio +- cover: Style transfer/regeneration with target audio as context +- extract: Extract specific track (e.g., vocals, drums) from audio +- lego: Generate specific track within a region + +Patches are applied at module import time. +""" + +import torch +import math +import functools +from . import logger + +# Flag to track if patches have been applied +_patches_applied = False + +# Task instructions from official ACE-Step 1.5 constants +TASK_INSTRUCTIONS = { + "text2music": "Fill the audio semantic mask based on the given conditions:", + "repaint": "Repaint the mask area based on the given conditions:", + "cover": "Generate audio semantic tokens based on the given conditions:", + "extract": "Extract the {TRACK_NAME} track from the audio:", + "extract_default": "Extract the track from the audio:", + "lego": "Generate the {TRACK_NAME} track based on the audio context:", + "lego_default": "Generate the track based on the audio context:", + "complete": "Complete the input track with {TRACK_CLASSES}:", + "complete_default": "Complete the input track:", +} + +# Valid track names for extract/lego tasks +TRACK_NAMES = [ + "woodwinds", "brass", "fx", "synth", "strings", "percussion", + "keyboard", "guitar", "bass", "drums", "backing_vocals", "vocals" +] + + +def get_task_instruction(task_type: str, track_name: str = None) -> str: + """ + Get the appropriate instruction for a task type. + + Args: + task_type: One of text2music, repaint, cover, extract, lego, complete + track_name: Required for extract/lego tasks (e.g., "vocals", "drums") + + Returns: + The instruction string for the task + """ + if task_type not in TASK_INSTRUCTIONS and f"{task_type}_default" not in TASK_INSTRUCTIONS: + raise ValueError(f"Unknown task type: {task_type}. Valid types: {list(TASK_INSTRUCTIONS.keys())}") + + instruction = TASK_INSTRUCTIONS.get(task_type, TASK_INSTRUCTIONS.get(f"{task_type}_default", "")) + + # Handle track name placeholder for extract/lego + if "{TRACK_NAME}" in instruction: + if track_name: + instruction = instruction.format(TRACK_NAME=track_name.upper()) + else: + # Fall back to default instruction without track name + instruction = TASK_INSTRUCTIONS.get(f"{task_type}_default", instruction.replace("{TRACK_NAME}", "")) + + return instruction + + +def _create_patched_ace_step15_forward(original_forward): + """ + Create a patched forward method for AceStepConditionGenerationModel. + + The patched version: + 1. Accepts chunk_masks, src_latents, is_covers, and precomputed_lm_hints_25Hz as kwargs + 2. Allows guiders to explicitly control is_covers flag for different task types + 3. Properly transposes user-provided tensors from (batch, channels, length) to (batch, length, channels) + + Task type behavior via is_covers: + - is_covers=0: Use provided src_latents directly (extend/repaint) + - is_covers=1: Use lm_hints from precomputed_lm_hints_25Hz or audio_codes (cover/extract) + - is_covers=None: Default behavior (src_latents = x) + + This enables extend, repaint, cover, extract, and lego task types. + """ + # One-time logging flag + _forward_logged = [False] + + @functools.wraps(original_forward) + def patched_forward(self, x, timestep, context, lyric_embed=None, refer_audio=None, + audio_codes=None, chunk_masks=None, src_latents=None, + is_covers=None, precomputed_lm_hints_25Hz=None, **kwargs): + text_attention_mask = None + lyric_attention_mask = None + refer_audio_order_mask = None + attention_mask = None + + # One-time diagnostic log for cover task + if not _forward_logged[0] and precomputed_lm_hints_25Hz is not None: + logger.debug(f"[ACE15_PATCHED_FORWARD] Cover mode detected") + logger.debug(f"[ACE15_PATCHED_FORWARD] x.shape (before transpose): {x.shape}") + logger.debug(f"[ACE15_PATCHED_FORWARD] is_covers: {is_covers}") + logger.debug(f"[ACE15_PATCHED_FORWARD] precomputed_lm_hints_25Hz.shape (before transpose): {precomputed_lm_hints_25Hz.shape}") + logger.debug(f"[ACE15_PATCHED_FORWARD] precomputed_lm_hints_25Hz stats: mean={precomputed_lm_hints_25Hz.mean():.4f}, std={precomputed_lm_hints_25Hz.std():.4f}") + _forward_logged[0] = True + + # is_covers can now be passed explicitly by guiders: + # - Cover/Extract tasks: is_covers=1 (use lm_hints from precomputed_lm_hints_25Hz) + # - Extend/Repaint tasks: is_covers=0 (use provided src_latents) + # - Default (not provided): fallback to old behavior + if is_covers is None and src_latents is not None: + # Backwards compatibility: default to 0 when src_latents provided but is_covers not specified + is_covers = torch.zeros((x.shape[0],), device=x.device, dtype=torch.long) + + lyric_hidden_states = lyric_embed + text_hidden_states = context + refer_audio_acoustic_hidden_states_packed = refer_audio.movedim(-1, -2) + + x = x.movedim(-1, -2) + + if refer_audio_order_mask is None: + refer_audio_order_mask = torch.zeros((x.shape[0],), device=x.device, dtype=torch.long) + + # Transpose user-provided tensors from ComfyUI format (batch, channels, length) + # to model format (batch, length, channels) + if src_latents is not None: + src_latents = src_latents.movedim(-1, -2) + if chunk_masks is not None: + chunk_masks = chunk_masks.movedim(-1, -2) + if precomputed_lm_hints_25Hz is not None: + # precomputed_lm_hints_25Hz comes in ComfyUI format [B, D, T], transpose to [B, T, D] + precomputed_lm_hints_25Hz = precomputed_lm_hints_25Hz.movedim(-1, -2) + # Log post-transpose shape (one-time) + if _forward_logged[0] and not hasattr(patched_forward, '_post_transpose_logged'): + logger.debug(f"[ACE15_PATCHED_FORWARD] precomputed_lm_hints_25Hz.shape (after transpose): {precomputed_lm_hints_25Hz.shape}") + logger.debug(f"[ACE15_PATCHED_FORWARD] x.shape (after transpose): {x.shape}") + logger.debug(f"[ACE15_PATCHED_FORWARD] src_latents.shape (after transpose): {src_latents.shape if src_latents is not None else 'None'}") + patched_forward._post_transpose_logged = True + + if src_latents is None and is_covers is None: + src_latents = x + + if chunk_masks is None: + chunk_masks = torch.ones_like(x) + + enc_hidden, enc_mask, context_latents = self.prepare_condition( + text_hidden_states, text_attention_mask, + lyric_hidden_states, lyric_attention_mask, + refer_audio_acoustic_hidden_states_packed, refer_audio_order_mask, + src_latents, chunk_masks, is_covers, + precomputed_lm_hints_25Hz=precomputed_lm_hints_25Hz, + audio_codes=audio_codes + ) + + out = self.decoder( + hidden_states=x, + timestep=timestep, + timestep_r=timestep, + attention_mask=attention_mask, + encoder_hidden_states=enc_hidden, + encoder_attention_mask=enc_mask, + context_latents=context_latents + ) + + return out.movedim(-1, -2) + + return patched_forward + + +def _create_patched_tokenizer(original_tokenize_with_weights): + """ + Create a patched tokenize_with_weights method for ACE15Tokenizer. + + The patched version: + 1. Accepts task_type kwarg to determine the instruction + 2. Accepts track_name kwarg for extract/lego tasks + 3. Uses appropriate instruction template based on task type + """ + @functools.wraps(original_tokenize_with_weights) + def patched_tokenize_with_weights(self, text, return_word_ids=False, **kwargs): + out = {} + lyrics = kwargs.get("lyrics", "") + bpm = kwargs.get("bpm", 120) + duration = kwargs.get("duration", 120) + keyscale = kwargs.get("keyscale", "C major") + timesignature = kwargs.get("timesignature", 2) + language = kwargs.get("language", "en") + seed = kwargs.get("seed", 0) + + # New kwargs for task type support + task_type = kwargs.get("task_type", "text2music") + track_name = kwargs.get("track_name", None) + + # Get appropriate instruction for task type + instruction = get_task_instruction(task_type, track_name) + + duration = math.ceil(duration) + meta_lm = 'bpm: {}\nduration: {}\nkeyscale: {}\ntimesignature: {}'.format(bpm, duration, keyscale, timesignature) + + # Use task-specific instruction in template + lm_template = "<|im_start|>system\n# Instruction\n{}\n\n<|im_end|>\n<|im_start|>user\n# Caption\n{}\n{}\n<|im_end|>\n<|im_start|>assistant\n\n{}\n\n\n<|im_end|>\n" + + meta_cap = '- bpm: {}\n- timesignature: {}\n- keyscale: {}\n- duration: {}\n'.format(bpm, timesignature, keyscale, duration) + out["lm_prompt"] = self.qwen3_06b.tokenize_with_weights(lm_template.format(instruction, text, lyrics, meta_lm), disable_weights=True) + out["lm_prompt_negative"] = self.qwen3_06b.tokenize_with_weights(lm_template.format(instruction, text, lyrics, ""), disable_weights=True) + + out["lyrics"] = self.qwen3_06b.tokenize_with_weights("# Languages\n{}\n\n# Lyric{}<|endoftext|><|endoftext|>".format(language, lyrics), return_word_ids, disable_weights=True, **kwargs) + out["qwen3_06b"] = self.qwen3_06b.tokenize_with_weights("# Instruction\n{}\n\n# Caption\n{}# Metas\n{}<|endoftext|>\n<|endoftext|>".format(instruction, text, meta_cap), return_word_ids, **kwargs) + out["lm_metadata"] = {"min_tokens": duration * 5, "seed": seed} + + # Store task info for downstream use + out["task_type"] = task_type + out["track_name"] = track_name + + return out + + return patched_tokenize_with_weights + + +def apply_patches(): + """ + Apply all ACE-Step 1.5 patches. + + This should be called once at module load time. + Safe to call multiple times (will only apply once). + """ + global _patches_applied + + if _patches_applied: + return + + try: + # Patch AceStepConditionGenerationModel.forward + from comfy.ldm.ace.ace_step15 import AceStepConditionGenerationModel + original_forward = AceStepConditionGenerationModel.forward + AceStepConditionGenerationModel.forward = _create_patched_ace_step15_forward(original_forward) + logger.info("[ACE-Step Patches] Patched AceStepConditionGenerationModel.forward") + except ImportError as e: + logger.info(f"[ACE-Step Patches] Warning: Could not patch ace_step15: {e}") + + try: + # Patch ACE15Tokenizer.tokenize_with_weights + from comfy.text_encoders.ace15 import ACE15Tokenizer + original_tokenize = ACE15Tokenizer.tokenize_with_weights + ACE15Tokenizer.tokenize_with_weights = _create_patched_tokenizer(original_tokenize) + logger.info("[ACE-Step Patches] Patched ACE15Tokenizer.tokenize_with_weights") + except ImportError as e: + logger.info(f"[ACE-Step Patches] Warning: Could not patch ace15 tokenizer: {e}") + + _patches_applied = True + logger.info("[ACE-Step Patches] All patches applied successfully") + + +def is_patched(): + """Check if patches have been applied.""" + return _patches_applied + + +# Apply patches when this module is imported +apply_patches() diff --git a/nodes/audio/__init__.py b/nodes/audio/__init__.py new file mode 100644 index 0000000..be6d2da --- /dev/null +++ b/nodes/audio/__init__.py @@ -0,0 +1 @@ +# Empty init file for audio package diff --git a/nodes/audio/audio_nodes_utility.py b/nodes/audio/audio_nodes_utility.py index d6716ae..210c940 100644 --- a/nodes/audio/audio_nodes_utility.py +++ b/nodes/audio/audio_nodes_utility.py @@ -203,6 +203,14 @@ class AudioSubtract(AudioUtility): #TODO: TOO SLOW @apply_tooltips class AudioInfo(AudioUtility): + # Krumhansl-Kessler key profiles for major and minor keys + MAJOR_PROFILE = np.array([6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88]) + MINOR_PROFILE = np.array([6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17]) + + # Note names matching the keyscale format in TextEncodeAceStepAudio1.5 + # Using sharps for detection, which covers all keys + NOTE_NAMES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"] + @classmethod def INPUT_TYPES(cls): return { @@ -212,47 +220,91 @@ class AudioInfo(AudioUtility): } } - RETURN_TYPES = ("INT", "INT", "INT", "INT", "INT", "FLOAT", "FLOAT", "FLOAT", "INT", "INT", "INT", "FLOAT", "FLOAT", "FLOAT", "STRING") + RETURN_TYPES = ("INT", "INT", "INT", "INT", "INT", "FLOAT", "FLOAT", "FLOAT", "INT", "INT", "INT", "INT", "FLOAT", "FLOAT", "FLOAT", "STRING", "COMBO") RETURN_NAMES = ( "total_frames", "frames_per_beat", "frames_per_bar", "frames_per_quarter", "frames_per_eighth", - "audio_duration", "beats_per_second", "detected_bpm", + "audio_duration", "beats_per_second", "detected_bpm", "detected_bpm_rounded", "sample_rate", "num_channels", "num_samples", - "max_amplitude", "mean_amplitude", "rms_amplitude", "bit_depth" + "max_amplitude", "mean_amplitude", "rms_amplitude", "bit_depth", + "detected_key" ) FUNCTION = "get_audio_info" + def _detect_key(self, audio_mono, sample_rate): + """Detect musical key using chromagram and Krumhansl-Kessler key profiles.""" + # Extract chromagram using Constant-Q Transform (better for pitch detection) + chromagram = librosa.feature.chroma_cqt(y=audio_mono, sr=sample_rate) + + # Average chroma values across time to get a single 12-element profile + chroma_profile = np.mean(chromagram, axis=1) + + # Normalize the profile + chroma_profile = chroma_profile / (np.linalg.norm(chroma_profile) + 1e-8) + + best_correlation = -1 + best_key = "C major" + + # Test all 24 keys (12 major + 12 minor) + for i in range(12): + # Rotate the key profiles to match each root note + major_rotated = np.roll(self.MAJOR_PROFILE, i) + minor_rotated = np.roll(self.MINOR_PROFILE, i) + + # Normalize profiles + major_rotated = major_rotated / (np.linalg.norm(major_rotated) + 1e-8) + minor_rotated = minor_rotated / (np.linalg.norm(minor_rotated) + 1e-8) + + # Calculate correlation with major key + major_corr = np.corrcoef(chroma_profile, major_rotated)[0, 1] + if major_corr > best_correlation: + best_correlation = major_corr + best_key = f"{self.NOTE_NAMES[i]} major" + + # Calculate correlation with minor key + minor_corr = np.corrcoef(chroma_profile, minor_rotated)[0, 1] + if minor_corr > best_correlation: + best_correlation = minor_corr + best_key = f"{self.NOTE_NAMES[i]} minor" + + return best_key + def get_audio_info(self, audio, frame_rate): # Get basic audio info waveform = audio['waveform'] sample_rate = audio['sample_rate'] - + # Calculate original audio info first num_channels = waveform.shape[1] if waveform.dim() > 2 else 1 num_samples = waveform.shape[-1] audio_duration = num_samples / sample_rate - + # Calculate total frames total_frames = int(audio_duration * frame_rate) - + # Detect BPM using librosa audio_mono = waveform.squeeze(0).mean(axis=0).cpu().numpy() tempo, _ = librosa.beat.beat_track(y=audio_mono, sr=sample_rate) + tempo_scalar = float(tempo.item() if hasattr(tempo, 'item') else tempo) + detected_bpm_rounded = int(round(tempo_scalar)) beats_per_second = tempo / 60.0 - + # Calculate frames per beat and musical divisions frames_per_beat = int(frame_rate / beats_per_second) frames_per_bar = frames_per_beat * 4 # Assuming 4/4 time signature frames_per_quarter = frames_per_beat frames_per_eighth = frames_per_beat // 2 - + # Calculate amplitude statistics max_amplitude = float(torch.max(torch.abs(waveform))) mean_amplitude = float(torch.mean(torch.abs(waveform))) rms_amplitude = float(torch.sqrt(torch.mean(waveform ** 2))) - + # Get bit depth from dtype bit_depth = str(waveform.dtype) - + + # Detect musical key + detected_key = self._detect_key(audio_mono, sample_rate) + return ( total_frames, frames_per_beat, @@ -262,13 +314,15 @@ class AudioInfo(AudioUtility): audio_duration, beats_per_second, tempo, # detected_bpm + detected_bpm_rounded, sample_rate, num_channels, num_samples, max_amplitude, mean_amplitude, rms_amplitude, - bit_depth + bit_depth, + detected_key ) @apply_tooltips diff --git a/pyproject.toml b/pyproject.toml index 8a84819..6a9be6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [project] name = "comfyui_ryanonyheinside" description = "Custom nodes introducing everything-reactivity and timelines to ComfyUI" -version = "2.0.12" +version = "2.1.0" license = {file = "LICENSE"} -dependencies = ["pygame","opencv-python>=4.10.0","scipy","torchaudio", "pillow","librosa>=0.10.2","pymunk==6.8.1","matplotlib","openunmix","mido","scikit-image"] +dependencies = ["pygame", "opencv-python>=4.5.0", "scipy", "torchaudio", "pillow", "librosa>=0.10.1", "pymunk>=6.8.0,<7.0.0", "matplotlib", "openunmix", "mido", "scikit-image"] [project.urls] Repository = "https://github.com/ryanontheinside/ComfyUI_RyanOnTheInside" diff --git a/readme.md b/readme.md index 89b24bd..19f10c6 100644 --- a/readme.md +++ b/readme.md @@ -12,7 +12,7 @@ - 🎵 **Audio & MIDI Processing**: Separate instruments, manipulate audio, work with ACEStep and other audio models - 🎆 **Particle Systems**: Create mesmerizing, fluid-like effects - 🌊 **Optical Flow**: Generate masks based on motion in videos -- 🎨 **ACEStep Native**: Native ComfyUI implementation for audio repaint and extend functionality +- 🎨 **ACE-Step 1.5 Native**: Native ComfyUI guiders for ACE-Step 1.5 — extend/repaint and cover - 🌊 **DEPTH Flow** 🌊: Use flex features to control depthflow nodes, adding parallax animations to your workflows! - 👤 **AdvancedLivePortrait** 👤: Use flex features to control facial animation expressions! - 🎨 **Advanced Controlnet** 🎨: Direct integration with ComfyUI-AdvancedControlnet! @@ -42,19 +42,6 @@ If you are not getting updates, or are having trouble updating from ComfyUI-Mana - **And much more!** -### ⚠️ Breaking Changes Notice -Due to ComfyUI's workflow loading mechanism, **existing workflows using these nodes ~~may~~ *will* break after updating**. I did consider this carefully, as I have yet to introduce breaking changes to this node system, but this extensive update neccesitated a complete overhaul. - -If you need to run an older workflow, you can revert to the previous version of these nodes by using the Manager, or by running this command in your ComfyUI_RyanOnTheInside directory: - -```bash -git checkout dab96492ac7d906368ac9c7a17cb0dbd670923d9 -``` - -To return to the latest version later, use: -```bash -git checkout main -``` @@ -135,10 +122,11 @@ Dynamic control over various aspects of your workflow: - Extract features from audio and MIDI for visual effects - Create audio-reactive animations and transformations -### 🎨 ACEStep Native -- **Repaint**: Selectively regenerate specific time ranges in existing audio -- **Extend**: Add new content before or after existing audio -- **Hybrid**: Combine repaint and extend operations simultaneously +### 🎨 ACE-Step 1.5 Native +Native ComfyUI guiders for the new [ACE-Step 1.5](https://github.com/ace-step/ACE-Step) music generation model. +- **Extend/Repaint**: Selectively regenerate specific regions or extend audio with new content +- **Cover**: Style transfer/regeneration using semantic guidance from source audio +- *Extract and Lego guiders coming soon* ### 🌊 Optical Flow - Generate masks based on movement in video sequences @@ -301,42 +289,41 @@ These audio and MIDI processing capabilities enable the creation of music-driven
-

ACEStep Native

 +

ACE-Step 1.5 Native

 -Native ComfyUI implementation of ACEStep audio repaint and extend functionality. This provides seamless integration with ComfyUI's existing sampling infrastructure to enable selective audio regeneration and extension without requiring custom types. +Native ComfyUI guiders for the [ACE-Step 1.5](https://github.com/ace-step/ACE-Step) music generation model. These integrate directly with ComfyUI's sampling infrastructure (SamplerCustomAdvanced) using model-level mask injection. -#### Core Functionality Nodes +#### Available Guiders -**ACEStep Repaint Guider** -Creates a guider for repainting specific audio regions. -- **Inputs**: Model, conditioning, source latents, time range, strength settings -- **Output**: GUIDER (for use with SamplerCustomAdvanced) +**ACE-Step 1.5 Edit Guider (Extend/Repaint)** +Unified guider for extend and/or repaint operations. +- Extend audio by adding new content before or after existing audio +- Repaint specific time regions while preserving the rest +- Combine both operations in a single pass +- Silence latent is automatically loaded from HuggingFace on first use -**ACEStep Extend Guider** -Creates a guider for extending audio before/after existing content. -- **Inputs**: Model, conditioning, source latents, extend times -- **Output**: GUIDER (for use with SamplerCustomAdvanced) - -**ACEStep Hybrid Guider** -Combines repaint and extend functionality in a single node. -- **Inputs**: All repaint/extend parameters with optional controls -- **Output**: GUIDER (for use with SamplerCustomAdvanced) +**ACE-Step 1.5 Cover Guider** +Style transfer/regeneration using semantic guidance from source audio. +- Uses semantic tokens extracted from source audio as structural guidance +- Generates new content with different style/timbre while preserving musical structure +- Semantic hints are auto-extracted or can be provided manually for advanced workflows #### Utility Nodes -- **ACEStep Analyze Latent**: Analyzes ACE latent properties for debugging -- **ACEStep Time Range**: Converts time ranges to frame indices -- **ACEStep Mask Visualizer**: Visualizes repaint masks for debugging -- **ACEStep Audio Post Processor**: Preserves original audio fidelity in unchanged regions +- **ACE-Step 1.5 Task Text Encode**: Task-aware text encoder with support for lyrics, BPM, key, time signature, and language +- **ACE-Step 1.5 Semantic Extractor**: Extract semantic tokens for use with Cover guider +- **ACE-Step 1.5 Semantic Hints Blend**: Blend semantic hints from multiple sources for mashups +- **ACEStep Analyze Latent**: Analyze audio latent properties +- **ACEStep Audio Post Processor**: Preserve original audio fidelity in unchanged regions -#### Basic Workflows +#### Coming Soon -**Repaint Workflow**: Replace audio content from 10-20 seconds with new generation. -See: [examples/acestep_repaint.json](examples/acestep_repaint.json) +- **Extract Guider**: Extract specific tracks (vocals, drums, bass, etc.) from source audio +- **Lego Guider**: Generate specific tracks within a time region -**Extend Workflow**: Add 10 seconds of new content after existing audio. -See: [examples/acestep_extend.json](examples/acestep_extend.json) +#### Legacy (ACE-Step 1.0) +The v1.0 Repaint, Extend, and Hybrid guiders remain available for use with ACE-Step 1.0 latents.
diff --git a/requirements.txt b/requirements.txt index 643259c..3f42091 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,10 @@ -opencv-python>=4.10.0 +opencv-python>=4.5.0 scipy pillow -librosa>=0.10.2 -pymunk==6.8.1 +librosa>=0.10.1 +pymunk>=6.8.0,<7.0.0 matplotlib openunmix mido -torchaudio pygame scikit-image \ No newline at end of file diff --git a/web/js/drawable_feature.js b/web/js/drawable_feature.js index 32ae987..518cec6 100644 --- a/web/js/drawable_feature.js +++ b/web/js/drawable_feature.js @@ -1,5 +1,4 @@ import { app } from "../../scripts/app.js"; -import { ComfyWidgets } from "../../scripts/widgets.js"; import { FeatureTemplates } from "./feature_templates.js";