diff --git a/examples/DynamiCrafter-CIL-testing_example_01.json b/examples/DynamiCrafter-CIL-testing_example_01.json new file mode 100644 index 0000000..7600131 --- /dev/null +++ b/examples/DynamiCrafter-CIL-testing_example_01.json @@ -0,0 +1,1240 @@ +{ + "last_node_id": 74, + "last_link_id": 185, + "nodes": [ + { + "id": 66, + "type": "AddLabel", + "pos": [ + 2050, + 160 + ], + "size": { + "0": 315, + "1": 274 + }, + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 163 + }, + { + "name": "caption", + "type": "STRING", + "link": null, + "widget": { + "name": "caption" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 164 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "AddLabel" + }, + "widgets_values": [ + 10, + 2, + 48, + 32, + "white", + "black", + "FreeMono.ttf", + "analytic_init_noise", + "up", + "" + ] + }, + { + "id": 1, + "type": "LoadImage", + "pos": [ + 490, + 200 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "Mona-Lisa-oil-wood-panel-Leonardo-da.webp", + "image" + ] + }, + { + "id": 52, + "type": "DownloadAndLoadDynamiCrafterModel", + "pos": [ + 534, + -178 + ], + "size": { + "0": 433.2352294921875, + "1": 106 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "DynCraft_model", + "type": "DCMODEL", + "links": [ + 138, + 152, + 155 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadDynamiCrafterModel" + }, + "widgets_values": [ + "DynamiCrafter-CIL-512-no-watermark-fp16.safetensors", + "auto", + false + ] + }, + { + "id": 64, + "type": "ImageConcanate", + "pos": [ + 2040, + -40 + ], + "size": { + "0": 315, + "1": 102 + }, + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "image1", + "type": "IMAGE", + "link": 162 + }, + { + "name": "image2", + "type": "IMAGE", + "link": 164 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 165 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImageConcanate" + }, + "widgets_values": [ + "right", + false + ] + }, + { + "id": 60, + "type": "DownloadAndLoadCLIPModel", + "pos": [ + 574, + -305 + ], + "size": { + "0": 371.02264404296875, + "1": 64.01405334472656 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "clip", + "type": "CLIP", + "links": [ + 147, + 148 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadCLIPModel" + }, + "widgets_values": [ + "stable-diffusion-2-1-clip-fp16.safetensors" + ] + }, + { + "id": 50, + "type": "CLIPTextEncode", + "pos": [ + 1025, + -65 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 148 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 141, + 158 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 29, + "type": "VHS_VideoCombine", + "pos": [ + 2410, + -390 + ], + "size": [ + 1550.3211669921875, + 853.9591693878174 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 165 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "AnimateDiff", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00005.mp4", + "subfolder": "", + "type": "temp", + "format": "video/h264-mp4" + } + } + } + }, + { + "id": 62, + "type": "DynamiCrafterLoadInitNoise", + "pos": [ + 495, + 1 + ], + "size": { + "0": 315, + "1": 122 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "DCMODEL", + "link": 152 + } + ], + "outputs": [ + { + "name": "init_noise", + "type": "DCNOISE", + "links": [ + 154 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 170 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 171 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "DynamiCrafterLoadInitNoise" + }, + "widgets_values": [ + 940, + true + ] + }, + { + "id": 5, + "type": "ImageResizeKJ", + "pos": [ + 973, + 239 + ], + "size": { + "0": 315, + "1": 242 + }, + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 2 + }, + { + "name": "get_image_size", + "type": "IMAGE", + "link": null + }, + { + "name": "width_input", + "type": "INT", + "link": 170, + "widget": { + "name": "width_input" + } + }, + { + "name": "height_input", + "type": "INT", + "link": 171, + "widget": { + "name": "height_input" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 172, + 173 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "height", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ImageResizeKJ" + }, + "widgets_values": [ + 512, + 320, + "lanczos", + false, + 64, + 0, + 0 + ] + }, + { + "id": 49, + "type": "CLIPTextEncode", + "pos": [ + 1028, + -310 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 147, + "slot_index": 0 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 140, + 157 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "nodding" + ] + }, + { + "id": 59, + "type": "DownloadAndLoadCLIPVisionModel", + "pos": [ + 622, + -439 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "clip_vision", + "type": "CLIP_VISION", + "links": [ + 146, + 156 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadCLIPVisionModel" + }, + "widgets_values": [ + "CLIP-ViT-H-fp16.safetensors" + ] + }, + { + "id": 65, + "type": "AddLabel", + "pos": [ + 2060, + -390 + ], + "size": { + "0": 315, + "1": 274 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 161 + }, + { + "name": "caption", + "type": "STRING", + "link": null, + "widget": { + "name": "caption" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 162 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "AddLabel" + }, + "widgets_values": [ + 10, + 2, + 48, + 32, + "white", + "black", + "FreeMono.ttf", + "baseline", + "up", + "" + ] + }, + { + "id": 71, + "type": "PrimitiveNode", + "pos": [ + 1332, + -736 + ], + "size": [ + 245.37998624877855, + 82 + ], + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 175, + 176 + ], + "widget": { + "name": "seed" + }, + "slot_index": 0 + } + ], + "title": "seed", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 619731667089950, + "fixed" + ] + }, + { + "id": 72, + "type": "PrimitiveNode", + "pos": [ + 1327, + -601 + ], + "size": [ + 252.17814718627847, + 82 + ], + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 178, + 179 + ], + "widget": { + "name": "steps" + } + } + ], + "title": "steps", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 26, + "fixed" + ] + }, + { + "id": 73, + "type": "PrimitiveNode", + "pos": [ + 1328, + -466 + ], + "size": { + "0": 210, + "1": 82 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "FLOAT", + "type": "FLOAT", + "links": [ + 181, + 182 + ], + "widget": { + "name": "cfg" + }, + "slot_index": 0 + } + ], + "title": "cfg", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 7, + "fixed" + ] + }, + { + "id": 63, + "type": "DynamiCrafterI2V", + "pos": [ + 1670, + -390 + ], + "size": [ + 315, + 462 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "DCMODEL", + "link": 155 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 156 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 157 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 158 + }, + { + "name": "image", + "type": "IMAGE", + "link": 173 + }, + { + "name": "image2", + "type": "IMAGE", + "link": null + }, + { + "name": "mask", + "type": "MASK", + "link": null + }, + { + "name": "init_noise", + "type": "DCNOISE", + "link": null, + "slot_index": 7 + }, + { + "name": "seed", + "type": "INT", + "link": 176, + "widget": { + "name": "seed" + } + }, + { + "name": "steps", + "type": "INT", + "link": 178, + "widget": { + "name": "steps" + }, + "slot_index": 9 + }, + { + "name": "cfg", + "type": "FLOAT", + "link": 182, + "widget": { + "name": "cfg" + } + }, + { + "name": "fs", + "type": "INT", + "link": 185, + "widget": { + "name": "fs" + } + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 161 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "last_image", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DynamiCrafterI2V" + }, + "widgets_values": [ + 26, + 7, + 1, + 16, + 619731667089950, + "fixed", + 24, + true, + "auto", + 16, + 4, + 0 + ] + }, + { + "id": 58, + "type": "DynamiCrafterI2V", + "pos": [ + 1670, + 150 + ], + "size": [ + 315, + 462 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "DCMODEL", + "link": 138 + }, + { + "name": "clip_vision", + "type": "CLIP_VISION", + "link": 146 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 140 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 141 + }, + { + "name": "image", + "type": "IMAGE", + "link": 172 + }, + { + "name": "image2", + "type": "IMAGE", + "link": null + }, + { + "name": "mask", + "type": "MASK", + "link": null + }, + { + "name": "init_noise", + "type": "DCNOISE", + "link": 154, + "slot_index": 7 + }, + { + "name": "seed", + "type": "INT", + "link": 175, + "widget": { + "name": "seed" + }, + "slot_index": 8 + }, + { + "name": "steps", + "type": "INT", + "link": 179, + "widget": { + "name": "steps" + }, + "slot_index": 9 + }, + { + "name": "cfg", + "type": "FLOAT", + "link": 181, + "widget": { + "name": "cfg" + }, + "slot_index": 10 + }, + { + "name": "fs", + "type": "INT", + "link": 184, + "widget": { + "name": "fs" + }, + "slot_index": 11 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 163 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "last_image", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DynamiCrafterI2V" + }, + "widgets_values": [ + 26, + 7, + 1, + 16, + 619731667089950, + "fixed", + 24, + true, + "auto", + 16, + 4, + 0 + ] + }, + { + "id": 74, + "type": "PrimitiveNode", + "pos": [ + 1614, + -727 + ], + "size": { + "0": 210, + "1": 82 + }, + "flags": {}, + "order": 7, + "mode": 0, + "outputs": [ + { + "name": "INT", + "type": "INT", + "links": [ + 184, + 185 + ], + "widget": { + "name": "fs" + }, + "slot_index": 0 + } + ], + "title": "fs", + "properties": { + "Run widget replace on values": false + }, + "widgets_values": [ + 24, + "fixed" + ] + } + ], + "links": [ + [ + 2, + 1, + 0, + 5, + 0, + "IMAGE" + ], + [ + 138, + 52, + 0, + 58, + 0, + "DCMODEL" + ], + [ + 140, + 49, + 0, + 58, + 2, + "CONDITIONING" + ], + [ + 141, + 50, + 0, + 58, + 3, + "CONDITIONING" + ], + [ + 146, + 59, + 0, + 58, + 1, + "CLIP_VISION" + ], + [ + 147, + 60, + 0, + 49, + 0, + "CLIP" + ], + [ + 148, + 60, + 0, + 50, + 0, + "CLIP" + ], + [ + 152, + 52, + 0, + 62, + 0, + "DCMODEL" + ], + [ + 154, + 62, + 0, + 58, + 7, + "DCNOISE" + ], + [ + 155, + 52, + 0, + 63, + 0, + "DCMODEL" + ], + [ + 156, + 59, + 0, + 63, + 1, + "CLIP_VISION" + ], + [ + 157, + 49, + 0, + 63, + 2, + "CONDITIONING" + ], + [ + 158, + 50, + 0, + 63, + 3, + "CONDITIONING" + ], + [ + 161, + 63, + 0, + 65, + 0, + "IMAGE" + ], + [ + 162, + 65, + 0, + 64, + 0, + "IMAGE" + ], + [ + 163, + 58, + 0, + 66, + 0, + "IMAGE" + ], + [ + 164, + 66, + 0, + 64, + 1, + "IMAGE" + ], + [ + 165, + 64, + 0, + 29, + 0, + "IMAGE" + ], + [ + 170, + 62, + 1, + 5, + 2, + "INT" + ], + [ + 171, + 62, + 2, + 5, + 3, + "INT" + ], + [ + 172, + 5, + 0, + 58, + 4, + "IMAGE" + ], + [ + 173, + 5, + 0, + 63, + 4, + "IMAGE" + ], + [ + 175, + 71, + 0, + 58, + 8, + "INT" + ], + [ + 176, + 71, + 0, + 63, + 8, + "INT" + ], + [ + 178, + 72, + 0, + 63, + 9, + "INT" + ], + [ + 179, + 72, + 0, + 58, + 9, + "INT" + ], + [ + 181, + 73, + 0, + 58, + 10, + "FLOAT" + ], + [ + 182, + 73, + 0, + 63, + 10, + "FLOAT" + ], + [ + 184, + 74, + 0, + 58, + 11, + "INT" + ], + [ + 185, + 74, + 0, + 63, + 11, + "INT" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650712, + "offset": { + "0": -357.3632507324219, + "1": 839.998046875 + } + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/init_noises/initial_noise_1024.safetensors b/init_noises/initial_noise_1024.safetensors new file mode 100644 index 0000000..0f3ebf4 Binary files /dev/null and b/init_noises/initial_noise_1024.safetensors differ diff --git a/init_noises/initial_noise_512.safetensors b/init_noises/initial_noise_512.safetensors new file mode 100644 index 0000000..4bf49cf Binary files /dev/null and b/init_noises/initial_noise_512.safetensors differ diff --git a/lvdm/models/ddpm3d.py b/lvdm/models/ddpm3d.py index 0083855..a9fef51 100644 --- a/lvdm/models/ddpm3d.py +++ b/lvdm/models/ddpm3d.py @@ -222,6 +222,9 @@ class DDPM(pl.LightningModule): variance = extract_into_tensor(1.0 - self.alphas_cumprod, t, x_start.shape) log_variance = extract_into_tensor(self.log_one_minus_alphas_cumprod, t, x_start.shape) return mean, variance, log_variance + + def get_sqrt_alpha_t_bar(self,x_start,t): + return extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) def predict_start_from_noise(self, x_t, t, noise): return ( @@ -703,6 +706,7 @@ class LatentVisualDiffusion(LatentDiffusion): super().__init__(*args, **kwargs) self._init_embedder(img_cond_stage_config, freeze_embedder) self.image_proj_model = instantiate_from_config(image_proj_stage_config) + def _init_embedder(self, config, freeze=True): embedder = instantiate_from_config(config) if freeze: diff --git a/lvdm/models/samplers/ddim.py b/lvdm/models/samplers/ddim.py index 15772c5..d4bb89f 100644 --- a/lvdm/models/samplers/ddim.py +++ b/lvdm/models/samplers/ddim.py @@ -27,9 +27,9 @@ class DDIMSampler(object): attr = attr.to(torch.device(device)) setattr(self, name, attr) - def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., verbose=True): + def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., ddpm_from=1000, verbose=True): self.ddim_timesteps = make_ddim_timesteps(ddim_discr_method=ddim_discretize, num_ddim_timesteps=ddim_num_steps, - num_ddpm_timesteps=self.ddpm_num_timesteps,verbose=verbose) + num_ddpm_timesteps=ddpm_from,verbose=verbose) alphas_cumprod = self.model.alphas_cumprod assert alphas_cumprod.shape[0] == self.ddpm_num_timesteps, 'alphas have to be defined for each timestep' to_torch = lambda x: x.clone().detach().to(torch.float32).to(self.model.device) @@ -89,7 +89,8 @@ class DDIMSampler(object): fs=None, timestep_spacing='uniform', #uniform_trailing for starting from last timestep guidance_rescale=0.0, - noise_multiplier=0, + noise_multiplier=1.0, + ddpm_from=1000, **kwargs ): @@ -107,7 +108,7 @@ class DDIMSampler(object): if conditioning.shape[0] != batch_size: print(f"Warning: Got {conditioning.shape[0]} conditionings but batch-size is {batch_size}") - self.make_schedule(ddim_num_steps=S, ddim_discretize=timestep_spacing, ddim_eta=eta, verbose=schedule_verbose) + self.make_schedule(ddim_num_steps=S, ddim_discretize=timestep_spacing, ddim_eta=eta, ddpm_from=ddpm_from, verbose=schedule_verbose) # make shape if len(shape) == 3: diff --git a/nodes.py b/nodes.py index f987e8c..d80bf0c 100644 --- a/nodes.py +++ b/nodes.py @@ -46,7 +46,8 @@ class DownloadAndLoadDynamiCrafterModel: "model": ( [ 'tooncrafter_512_interp-fp16.safetensors', 'dynamicrafter_512_interp_v1_bf16.safetensors', - 'dynamicrafter_1024_v1_bf16.safetensors' + 'dynamicrafter_1024_v1_bf16.safetensors', + 'DynamiCrafter-CIL-512-no-watermark-fp16.safetensors', ], { "default": 'tooncrafter_512_interp-fp16.safetensors' @@ -133,7 +134,12 @@ class DownloadAndLoadDynamiCrafterModel: if fp8_unet: self.model.model.diffusion_model = self.model.model.diffusion_model.to(torch.float8_e4m3fn) print(f"Model using dtype: {self.model.dtype}") - return (self.model,) + + dcmodel = { + 'model': self.model, + 'model_name': model, + } + return (dcmodel,) class DownloadAndLoadCLIPModel: @classmethod @@ -298,7 +304,11 @@ class DynamiCrafterModelLoader: if fp8_unet: self.model.model.diffusion_model = self.model.model.diffusion_model.to(torch.float8_e4m3fn) print(f"Model using dtype: {self.model.dtype}") - return (self.model,) + dcmodel = { + 'model': self.model, + 'model_name': ckpt_name, + } + return (dcmodel,) class DynamiCrafterI2V: @classmethod @@ -332,7 +342,8 @@ class DynamiCrafterI2V: "mask": ("MASK",), "frame_window_size": ("INT", {"default": 16, "min": 1, "max": 200, "step": 1}), "frame_window_stride": ("INT", {"default": 4, "min": 1, "max": 200, "step": 1}), - "augmentation_level": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.0001}) + "augmentation_level": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 10.0, "step": 0.0001}), + "init_noise": ("DCNOISE",), } } @@ -342,27 +353,29 @@ class DynamiCrafterI2V: CATEGORY = "DynamiCrafterWrapper" def process(self, model, image, clip_vision, positive, negative, cfg, steps, eta, seed, fs, keep_model_loaded, - frames, vae_dtype, frame_window_size=16, frame_window_stride=4, mask=None, image2=None, augmentation_level=0): + frames, vae_dtype, frame_window_size=16, frame_window_stride=4, mask=None, image2=None, augmentation_level=0, init_noise=None): device = mm.get_torch_device() offload_device = mm.unet_offload_device() mm.unload_all_models() mm.soft_empty_cache() + self.model = model['model'] + torch.manual_seed(seed) - dtype = model.dtype + dtype = self.model.dtype if vae_dtype == "auto": try: if mm.should_use_bf16(): - model.first_stage_model.to(convert_dtype('bf16')) + self.model.first_stage_model.to(convert_dtype('bf16')) else: - model.first_stage_model.to(convert_dtype('fp32')) + self.model.first_stage_model.to(convert_dtype('fp32')) except: raise AttributeError("ComfyUI version too old, can't autodetect properly. Set your dtype manually.") else: - model.first_stage_model.to(convert_dtype(vae_dtype)) - print(f"VAE using dtype: {model.first_stage_model.dtype}") + self.model.first_stage_model.to(convert_dtype(vae_dtype)) + print(f"VAE using dtype: {self.model.first_stage_model.dtype}") - self.model = model + self.model.to(device) autocast_condition = (dtype != torch.float32) and not comfy.model_management.is_device_mps(device) with torch.autocast(comfy.model_management.get_autocast_device(device), dtype=dtype) if autocast_condition else nullcontext(): @@ -461,6 +474,20 @@ class DynamiCrafterI2V: mask = mask.permute(0, 2, 1, 3, 4) mask = torch.where(mask < 1.0, torch.tensor(0.0, device=device, dtype=dtype), torch.tensor(1.0, device=device, dtype=dtype)) + if init_noise is not None: + init = init_noise['noise'].to(dtype).to(device) + timestep_spacing = "uniform_trailing" + guidance_rescale = 0.0 + ddpm_from = init_noise['M'] + + if noise_shape[2] % init.shape[2] == 0: + init = init.repeat(1, 1, noise_shape[2] // init.shape[2], 1, 1) + else: + raise ValueError("The target dimension size is not an integral multiple of the original dimension size.") + else: + init = None + ddpm_from = 1000 + #inference ddim_sampler = DDIMSampler(self.model) samples, _ = ddim_sampler.sample(S=steps, @@ -473,7 +500,7 @@ class DynamiCrafterI2V: eta=eta, temporal_length=noise_shape[2], conditional_guidance_scale_temporal=None, - x_T=None, + x_T=init, fs=fs, timestep_spacing=timestep_spacing, guidance_rescale=guidance_rescale, @@ -481,7 +508,9 @@ class DynamiCrafterI2V: mask=mask, x0=img_tensor_repeat.clone() if mask is not None else None, frame_window_size = frame_window_size, - frame_window_stride = frame_window_stride + frame_window_stride = frame_window_stride, + noise_multiplier=1.0, + ddpm_from=ddpm_from ) assert not torch.isnan(samples).any().item(), "Resulting tensor containts NaNs. I'm unsure why this happens, changing step count and/or image dimensions might help." @@ -509,6 +538,56 @@ class DynamiCrafterI2V: video = F.interpolate(video.permute(0, 3, 1, 2), size=(final_H, final_W), mode="bicubic").permute(0, 2, 3, 1) last_image = video[-1].unsqueeze(0) return (video, last_image) + +class DynamiCrafterLoadInitNoise: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "model": ("DCMODEL",), + "M": ("INT", {"default": 1000, "min": 1, "max": 1000, "step": 1}), + "analytic_init": ("BOOLEAN", {"default": True}), + }, + } + + RETURN_TYPES = ("DCNOISE", "INT", "INT",) + RETURN_NAMES = ("init_noise", "width", "height",) + FUNCTION = "load" + CATEGORY = "DynamiCrafterWrapper" + + def load(self, model, M, analytic_init): + device = mm.get_torch_device() + + model_name = model['model_name'] + if '512' in model_name: + analytic_noise = "initial_noise_512.safetensors" + elif '1024' in model_name: + analytic_noise = "initial_noise_1024.safetensors" + else: + print("Can't find matching init_noise for model: ", model_name) + model_path = os.path.join(script_directory, 'init_noises', analytic_noise) + + # Analytic-Init:load initial noise + #dic=torch.load(model_path) + dic = comfy.utils.load_torch_file(model_path) + expectation_X_0=dic["Expectation_X0"].to(device) + tr_Cov_d=dic["Tr_Cov_d"].to(device) + sqrt_alpha_t=model['model'].get_sqrt_alpha_t_bar(expectation_X_0,torch.tensor([M-1]).to(device)) + mu_p=sqrt_alpha_t*expectation_X_0 + alpha_t=sqrt_alpha_t**2 + sigma_p=torch.sqrt(1-alpha_t + alpha_t*tr_Cov_d) + eps=torch.randn_like(mu_p) + + if analytic_init: + init=mu_p+sigma_p*eps + else : + init=torch.randn_like(mu_p) + print("init noise shape: ",init.shape) + + init_noise = {"noise": init, "M": M} + width = init.shape[4] * 8 + height = init.shape[3] * 8 + + return (init_noise, width, height) class ToonCrafterInterpolation: @classmethod @@ -555,18 +634,21 @@ class ToonCrafterInterpolation: mm.soft_empty_cache() torch.manual_seed(seed) - dtype = model.dtype + + self.model = model['model'] + + dtype = self.model.dtype if vae_dtype == "auto": try: if mm.should_use_bf16(): - model.first_stage_model.to(convert_dtype('bf16')) + self.model.first_stage_model.to(convert_dtype('bf16')) else: - model.first_stage_model.to(convert_dtype('fp32')) + self.model.first_stage_model.to(convert_dtype('fp32')) except: raise AttributeError("ComfyUI version too old, can't autodetect properly. Set your dtype manually.") else: - model.first_stage_model.to(convert_dtype(vae_dtype)) - print(f"VAE using dtype: {model.first_stage_model.dtype}") + self.model.first_stage_model.to(convert_dtype(vae_dtype)) + print(f"VAE using dtype: {self.model.first_stage_model.dtype}") images = images.permute(0, 3, 1, 2).to(dtype).to(device) @@ -579,7 +661,6 @@ class ToonCrafterInterpolation: if orig_H % 64 != 0 or orig_W % 64 != 0: images = F.interpolate(images, size=(H, W), mode="bicubic") - self.model = model self.model.to(device) out = [] @@ -755,7 +836,7 @@ class ToonCrafterDecode: device = mm.get_torch_device() mm.unload_all_models() mm.soft_empty_cache() - + model = model['model'] samples = latent["samples"] num_samples = samples.shape[0] samples = samples * 0.18215 @@ -857,20 +938,20 @@ class DynamiCrafterBatchInterpolation: torch.manual_seed(seed) dtype = model.dtype + self.model = model['model'] if vae_dtype == "auto": try: if mm.should_use_bf16(): - model.first_stage_model.to(convert_dtype('bf16')) + self.model.first_stage_model.to(convert_dtype('bf16')) else: - model.first_stage_model.to(convert_dtype('fp32')) + self.model.first_stage_model.to(convert_dtype('fp32')) except: raise AttributeError("ComfyUI version too old, can't autodetect properly. Set your dtype manually.") else: - model.first_stage_model.to(convert_dtype(vae_dtype)) - print(f"VAE using dtype: {model.first_stage_model.dtype}") - - self.model = model + self.model.first_stage_model.to(convert_dtype(vae_dtype)) + print(f"VAE using dtype: {self.model.first_stage_model.dtype}") + self.model.to(device) images = images * 2 - 1 images = images.permute(0, 3, 1, 2).to(dtype).to(device) @@ -1020,7 +1101,8 @@ NODE_CLASS_MAPPINGS = { "ToonCrafterDecode": ToonCrafterDecode, "DownloadAndLoadDynamiCrafterModel": DownloadAndLoadDynamiCrafterModel, "DownloadAndLoadCLIPModel": DownloadAndLoadCLIPModel, - "DownloadAndLoadCLIPVisionModel": DownloadAndLoadCLIPVisionModel + "DownloadAndLoadCLIPVisionModel": DownloadAndLoadCLIPVisionModel, + "DynamiCrafterLoadInitNoise": DynamiCrafterLoadInitNoise } NODE_DISPLAY_NAME_MAPPINGS = { @@ -1031,5 +1113,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ToonCrafterDecode": "ToonCrafterDecode", "DownloadAndLoadDynamiCrafterModel": "DownloadAndLoadDynamiCrafterModel", "DownloadAndLoadCLIPModel": "DownloadAndLoadCLIPModel", - "DownloadAndLoadCLIPVisionModel": "DownloadAndLoadCLIPVisionModel" + "DownloadAndLoadCLIPVisionModel": "DownloadAndLoadCLIPVisionModel", + "DynamiCrafterLoadInitNoise": "DynamiCrafterLoadInitNoise" }