idk whats even in this but i'll document it and push it later™️

Implement IPNDM_VAPP (IPNDM_V with ancestral sampling and CFGPP)
Implement STRIKE (A heavily modified Euler with denoised history and 'full ancestral sampling')

README.md

@@ -8,6 +8,9 @@
 * LCM Custom Noise (Supports different types of noise other than generic gaussian)
 * DPMPP 3M SDE with Dynamic ETA (Anneals down towards a minimum eta via a cosine curve)
 * Supreme (Many extra functionalities and step methods available)
+* SENS (SDE-Endowed Nimble Sampler. Based off of DPM-Solver++(2M) SDE and DPM-Solver++(3M) SDE. R-SDE for reversible SDE, T-SDE for tertiary SDE.)
+* IPNDM VAPP (IPNDM_V with ancestral sampling and (somewhat) CFGPP)
+* STRIKE (Stochastic, Temporal, Reversible, Improvised K-Diffusion Experiment. Based off of Euler A with an ancestral and SDE twist.)
 
 ### Currently included extra K-Sampling nodes:
 * SamplerCustomNoise (Supports custom noises other than gaussian noise for init noise)
@@ -19,6 +22,14 @@
 * ScaledCFGGuider: Samples the two conditionings, then adds it using a method similar to "Add Trained Difference" from merging models.
 * ImageAssistedCFGGuider: Samples the conditioning, then adds in the latent image using vector projection onto the CFG. Image latent ought to be of the same size as the diffusion latent.
 
+### Currently included extra schedulers:
+* SimpleExponentialScheduler: Using Simple scheduler as a base, apply an exponential decay. (Works with ZSNR)
+* KLOptimalScheduler: KL Optimal/'Gaussian' scheduler, may be good at low step counts.
+* SimpleKLOptimalScheduler: KL Optimal/'Gaussian' scheduler, but using Simple scheduler as a base to work off of. (Works with ZSNR)
+
+### Currently included extra noise types:
+* Immiscible Noise: Aligns the noise with the given image latent. For ancestral sampling, we utilize `cond_denoised` as the reference image latent.
+
 #### Supreme Sampler features:
 * step_method: You have the ability to choose your own step method with this sampler! Optionally, there's a dynamic step method, which chooses the appropriate order based on the calculated error between steps, allowing you to obtain higher quality when it matters in the sampling process. Defaults to **(Euler)**.
 * centralization: Subtracts mean from the denoised latent. This can lead to perceptually sharper results, though may change the perceivable brightness of the image. Conservatively defaults to **(0.05)**.

__init__.py

@@ -15,6 +15,9 @@ NODE_CLASS_MAPPINGS = {
     "ScaledCFGGuider": nodes.ScaledCFGGuider,
     "WarmupDecayCFGGuider": nodes.WarmupDecayCFGGuider,
     "MegaCFGGuider": nodes.MegaCFGGuider,
+    "APGGuider": nodes.APGGuider,
+    ### Noise
+    "ImmiscibleNoise": nodes.ImmiscibleNoise,
     ## Samplers
     "SamplerRES_Momentumized": nodes.SamplerRES_MOMENTUMIZED,
     "SamplerDPMPP_DualSDE_Momentumized": nodes.SamplerDPMPP_DUALSDE_MOMENTUMIZED,
@@ -24,7 +27,16 @@ NODE_CLASS_MAPPINGS = {
     "SamplerEulerAncestralDancing_Experimental": nodes.SamplerEULER_ANCESTRAL_DANCING,
     "SamplerDPMPP_3M_SDE_DynETA": nodes.SamplerDPMPP_3M_SDE_DYN_ETA,
     "SamplerSupreme": nodes.SamplerSUPREME,
+    "SamplerSENS": nodes.SamplerSENS,
+    "SamplerIPNDM_VAPP": nodes.SamplerIPNDM_VAPP,
+    "SamplerSHIDS": nodes.SamplerSHIDS,
+    "SamplerDPMPP_2M_SDE_EMA": nodes.SamplerDPMPP_2M_SDE_EMA,
+    "SamplerBiScope": nodes.SamplerBiScope,
+    "SamplerEuler_G": nodes.SamplerEuler_G,
+    "SamplerLeaping_Euler": nodes.SamplerLeaping_Euler,
     ### Schedulers
     "SimpleExponentialScheduler": nodes.SimpleExponentialScheduler,
+    "KLOptimalScheduler": nodes.KLOptimalScheduler,
+    "SimpleKLOptimalScheduler": nodes.SimpleKLOptimalScheduler,
 }
 __all__ = ['NODE_CLASS_MAPPINGS']

nodes.py

@@ -1,5 +1,5 @@
 from .other_samplers.refined_exp_solver import sample_refined_exp_s
-from .extra_samplers import get_noise_sampler_names, prepare_noise
+from .extra_samplers import get_noise_sampler_names, get_immiscible_noise_sampler_names, prepare_noise, make_immiscible
 
 import comfy.samplers
 import comfy.sample
@@ -11,6 +11,7 @@ import latent_preview
 import torch
 import math
 from tqdm.auto import trange
+import numpy as np
 
 import kornia
 
@@ -156,19 +157,25 @@ class SamplerSUPREME:
         NOISE_MODULATION_TYPES=["none", "intensity", "frequency", "spectral_signum"]
         return {"required":
                     {"noise_sampler_type": (get_noise_sampler_names(),),
-                     "step_method": (STEP_METHODS, ),
-                     "substep_method": (SUBSTEP_METHODS, ),
+                     "step_method": (STEP_METHODS, {"default": "euler"}),
+                     "substep_method": (SUBSTEP_METHODS, {"default": "euler"}),
+                     "warmup_method": (SUBSTEP_METHODS, {"default": "euler"}),
                      "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
-                     "centralization": ("FLOAT", {"default": 0.02, "min": -1.0, "max": 1.0, "step":0.01}),
-                     "normalization": ("FLOAT", {"default": 0.01, "min": -1.0, "max": 1.0, "step":0.01}),
-                     "edge_enhancement": ("FLOAT", {"default": 0.05, "min": -100.0, "max": 100.0, "step":0.01}),
-                     "perphist": ("FLOAT", {"default": 0, "min": -5.0, "max": 5.0, "step":0.01}),
+                     "centralization": ("FLOAT", {"default": 0.00, "min": -1.0, "max": 1.0, "step":0.01}),
+                     "normalization": ("FLOAT", {"default": 0.00, "min": -1.0, "max": 1.0, "step":0.01}),
+                     "edge_enhancement": ("FLOAT", {"default": 0.00, "min": -100.0, "max": 100.0, "step":0.01}),
+                     "perphist": ("FLOAT", {"default": 0.25, "min": -1.0, "max": 1.0, "step":0.01}),
                      "substeps": ("INT", {"default": 2, "min": 1, "max": 100, "step":1}),
                      "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
-                     "noise_modulation": (NOISE_MODULATION_TYPES, {"default": "intensity"}),
-                     "modulation_strength": ("FLOAT", {"default": 2.0, "min": -100.0, "max": 100.0, "step":0.01}),
+                     "noise_modulation": (NOISE_MODULATION_TYPES, {"default": "none"}),
+                     "modulation_strength": ("FLOAT", {"default": 2., "min": -100.0, "max": 100.0, "step":0.01}),
                      "modulation_dims": ("INT", {"default": 3, "min": 1, "max": 3, "step":1}),
                      "reversible_eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "dyneta": ("BOOLEAN", {"default": True}),
+                     "reversible_dyneta": ("BOOLEAN", {"default": True}),
+                     "enable_free_reverse": ("BOOLEAN", {"default": True}),
+                     "free_reverse_eta": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "free_reverse_dyneta": ("BOOLEAN", {"default": True}),
                       }
                }
     RETURN_TYPES = ("SAMPLER",)
@@ -176,10 +183,257 @@ class SamplerSUPREME:
 
     FUNCTION = "get_sampler"
 
-    def get_sampler(self, noise_sampler_type, step_method, substep_method, eta, centralization, normalization, edge_enhancement, perphist, substeps, noise_modulation, modulation_strength, modulation_dims, reversible_eta, s_noise):
-        sampler = comfy.samplers.ksampler("supreme", {"noise_sampler_type": noise_sampler_type, "step_method": step_method, "eta": eta, "centralization": centralization, "normalization": normalization, "edge_enhancement": edge_enhancement, "perphist": perphist, "substeps": substeps, "substep_method": substep_method, "noise_modulation": noise_modulation, "modulation_strength": modulation_strength, "modulation_dims": modulation_dims, "reversible_eta": reversible_eta, "s_noise": s_noise})
+    def get_sampler(self, noise_sampler_type, step_method, substep_method, warmup_method, eta, centralization, normalization, edge_enhancement, perphist, substeps, noise_modulation, modulation_strength, modulation_dims, reversible_eta, dyneta, reversible_dyneta, enable_free_reverse, free_reverse_eta, free_reverse_dyneta, s_noise):
+        sampler = comfy.samplers.ksampler("supreme", {"noise_sampler_type": noise_sampler_type, "step_method": step_method, "eta": eta, "centralization": centralization, "normalization": normalization, "edge_enhancement": edge_enhancement, "perphist": perphist, "substeps": substeps, "substep_method": substep_method, "warmup_method": warmup_method, "noise_modulation": noise_modulation, "modulation_strength": modulation_strength, "modulation_dims": modulation_dims, "reversible_eta": reversible_eta, "dyneta": dyneta, "reversible_dyneta": reversible_dyneta, "enable_free_reverse": enable_free_reverse, "free_reverse_eta": free_reverse_eta, "free_reverse_dyneta": free_reverse_dyneta, "s_noise": s_noise})
         return (sampler, )
 
+# SENS
+class SamplerSENS:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="brownian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "rsde_eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "tsde_eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, rsde_eta, tsde_eta, s_noise):
+        sampler = comfy.samplers.ksampler("sens", {"noise_sampler_type": noise_sampler_type, "eta": eta, "rsde_eta": rsde_eta, "tsde_eta": tsde_eta, "s_noise": s_noise})
+        return (sampler, )
+
+# IPNDM_V Ancestral CFG++
+class SamplerIPNDM_VAPP:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="brownian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "max_order": ("INT", {"default": 4, "min": 1, "max": 4, "step":1}),
+                     "pp_guidance": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, s_noise, max_order, pp_guidance):
+        sampler = comfy.samplers.ksampler("ipndm_vapp", {"noise_sampler_type": noise_sampler_type, "eta": eta, "s_noise": s_noise, "max_order": max_order, "pp_guidance": pp_guidance})
+        return (sampler, )
+
+# SHIDS (Stochastic Historical Improvised Sampling)
+class SamplerSHIDS:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="gaussian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "order": ("INT", {"default": 16, "min": 1, "max": 100, "step":1}),
+                     "eta_order": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "solver_method": (["weighted_projection", "qr_decomposition", "svd_lowrank", "svd"], {"default": "weighted_projection"}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, s_noise, order, eta_order, solver_method):
+        sampler = comfy.samplers.ksampler("SHIDS", {"noise_sampler_type": noise_sampler_type, "eta": eta, "s_noise": s_noise, "order": order, "eta_order": eta_order, "solver_method": solver_method})
+        return (sampler, )
+
+# EMA DPM++ 2M SDE (Compass Optimizer-like implementation)
+class SamplerDPMPP_2M_SDE_EMA:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="brownian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "amp_fac": ("FLOAT", {"default": 2.0, "min": -100.0, "max": 100.0, "step":0.01}),
+                     "beta1": ("FLOAT", {"default": 0.8, "min": 0.0, "max": 0.999, "step":0.01}),
+                     "beta2": ("FLOAT", {"default": 0.95, "min": 0.0, "max": 0.999, "step":0.01}),
+                     "weight_decay": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "centralization": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step":0.01}),
+                     "normalization": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, s_noise, amp_fac, beta1, beta2, weight_decay, centralization, normalization):
+        sampler = comfy.samplers.ksampler("dpmpp_2m_sde_ema", {"noise_sampler_type": noise_sampler_type, "eta": eta, "s_noise": s_noise, "amp_fac": amp_fac, "beta1": beta1, "beta2": beta2, "weight_decay": weight_decay, "centralization": centralization, "normalization": normalization})
+        return (sampler, )
+
+class SamplerEuler_3EMA:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="gaussian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "amp_fac": ("FLOAT", {"default": 2.0, "min": -100.0, "max": 100, "step":0.01}),
+                     "smoothing_fac": ("FLOAT", {"default": 0.8, "min": 0.0, "max": 0.999, "step":0.01}),
+                     "ema_fac": ("FLOAT", {"default": 0.75, "min": 0.0, "max": 0.999, "step":0.01}),
+                     "beta": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 0.999, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, s_noise, amp_fac, smoothing_fac, ema_fac, beta):
+        sampler = comfy.samplers.ksampler("euler_3ema", {"noise_sampler_type": noise_sampler_type, "eta": eta, "s_noise": s_noise, "amp_fac": amp_fac, "smoothing_fac": smoothing_fac, "ema_fac": ema_fac, "beta": beta})
+        return (sampler, )
+
+class SamplerScope:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="gaussian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "amp_fac": ("FLOAT", {"default": 2.0, "min": -100.0, "max": 100, "step":0.01}),
+                     "smoothing_fac": ("FLOAT", {"default": 0.8, "min": 0.0, "max": 0.999, "step":0.01}),
+                     "ema_fac": ("FLOAT", {"default": 0.75, "min": 0.0, "max": 0.999, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, s_noise, amp_fac, smoothing_fac, ema_fac):
+        sampler = comfy.samplers.ksampler("scope", {"noise_sampler_type": noise_sampler_type, "eta": eta, "s_noise": s_noise, "amp_fac": amp_fac, "smoothing_fac": smoothing_fac, "ema_fac": ema_fac})
+        return (sampler, )
+
+class SamplerBiScope:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="gaussian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "amp_fac": ("FLOAT", {"default": 2.0, "min": -100.0, "max": 100, "step":0.01}),
+                     "local_smoothing_fac": ("INT", {"default": 4, "min": 1, "max": 100, "step":1}),
+                     "smoothing_fac": ("FLOAT", {"default": 0.8, "min": 0.0, "max": 0.999, "step":0.01}),
+                     "ema_fac": ("FLOAT", {"default": 0.75, "min": 0.0, "max": 0.999, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, s_noise, amp_fac, local_smoothing_fac, smoothing_fac, ema_fac):
+        sampler = comfy.samplers.ksampler("biscope", {"noise_sampler_type": noise_sampler_type, "eta": eta, "s_noise": s_noise, "amp_fac": amp_fac, "local_smoothing_fac": local_smoothing_fac, "smoothing_fac": smoothing_fac, "ema_fac": ema_fac})
+        return (sampler, )
+
+class SamplerEuler_G:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="gaussian"), ),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "g_eta": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step":0.01}),
+                     "sigma": ("FLOAT", {"default": 1, "min": 0.01, "max": 100.0, "step":0.01}),
+                     "order": ("INT", {"default": 3, "min": 3, "max": 100, "step":1}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, eta, g_eta, sigma, order, s_noise):
+        sampler = comfy.samplers.ksampler("euler_g", {"noise_sampler_type": noise_sampler_type, "eta": eta, "g_eta": g_eta, "sigma": sigma, "order": order, "s_noise": s_noise})
+        return (sampler, )
+
+class SamplerLeaping_Euler:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"noise_sampler_type": (get_noise_sampler_names(default="gaussian"), ),
+                     "leap": ("INT", {"default": 1, "min": 1, "max": 8, "step":1}),
+                     "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.01}),
+                     "s_noise": ("FLOAT", {"default": 1, "min": 0.0, "max": 100.0, "step":0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SAMPLER",)
+    CATEGORY = "sampling/custom_sampling/samplers"
+
+    FUNCTION = "get_sampler"
+
+    def get_sampler(self, noise_sampler_type, leap, eta, s_noise):
+        sampler = comfy.samplers.ksampler("leaping_euler", {"noise_sampler_type": noise_sampler_type, "leap": leap, "eta": eta, "s_noise": s_noise})
+        return (sampler, )
+
+### Noise
+
+class Noise_ImmiscibleNoise:
+    def __init__(self, noise_type, seed, image_scaling, latent_image, n_latents = 1024):
+        self.noise_type = noise_type
+        self.seed = seed
+        self.image_scaling = image_scaling
+        self.latent_image = latent_image
+        self.n_latents = n_latents
+
+    def generate_noise(self, input_latent):
+        latent_image = input_latent["samples"]
+        batch_inds = input_latent["batch_index"] if "batch_index" in input_latent else None
+        generator = torch.manual_seed(self.seed)
+        if batch_inds is None:
+            gauss = torch.randn_like(latent_image)
+            noise = make_immiscible(noise_func=self.noise_type, immiscible_latents=self.n_latents)(latent_image if self.latent_image is None else self.latent_image["samples"])
+            noise = gauss * (1.0 - self.image_scaling) + noise * self.image_scaling
+            return noise
+            #return torch.randn(latent_image.size(), dtype=latent_image.dtype, layout=latent_image.layout, generator=generator, device="cpu")
+        
+        unique_inds, inverse = np.unique(batch_inds, return_inverse=True)
+        noises = []
+        for i in range(unique_inds[-1]+1):
+            gauss = torch.randn_like(latent_image)
+            noise = make_immiscible(noise_func=self.noise_type, immiscible_latents=self.n_latents)(latent_image if self.latent_image is None else self.latent_image["samples"])
+            noise = gauss * (1.0 - self.image_scaling) + noise * self.image_scaling
+            if i in unique_inds:
+                noises.append(noise)
+        noises = [noises[i] for i in inverse]
+        noises = torch.cat(noises, axis=0)
+        return noises
+
+from comfy_extras.nodes_custom_sampler import DisableNoise
+class ImmiscibleNoise(DisableNoise):
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {
+                    "noise_type": (get_immiscible_noise_sampler_names(), ),
+                    "n_latents": ("INT", {"default": 1024, "min": 1, "max": 16384, "step":1}),
+                    "noise_seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+                    },
+                "optional":
+                    {
+                    "image_scaling": ("FLOAT", {"default": 1.0, "min": -1000.0, "max": 1000.0, "step":0.01, "round": 0.001}),
+                    "latent_image": ("LATENT", ),
+                    }
+                }
+
+    def get_noise(self, noise_type, n_latents, noise_seed, image_scaling, latent_image):
+        return (Noise_ImmiscibleNoise(noise_type, noise_seed, image_scaling, latent_image, n_latents),)
+
 ### Schedulers
 from .extra_samplers import get_sigmas_simple_exponential
 class SimpleExponentialScheduler:
@@ -192,7 +446,7 @@ class SimpleExponentialScheduler:
                       }
                }
     RETURN_TYPES = ("SIGMAS",)
-    CATEGORY = "clybNodes/schedulers"
+    CATEGORY = "sampling/custom_sampling/schedulers"
 
     FUNCTION = "get_sigmas"
 
@@ -205,6 +459,54 @@ class SimpleExponentialScheduler:
         sigmas = sigmas[-(steps + 1):]
         return (sigmas, )
 
+from .extra_samplers import get_sigmas_simple_kl_optimal
+class SimpleKLOptimalScheduler:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"model": ("MODEL",),
+                     "steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
+                     "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SIGMAS",)
+    CATEGORY = "sampling/custom_sampling/schedulers"
+
+    FUNCTION = "get_sigmas"
+
+    def get_sigmas(self, model, steps, denoise):
+        total_steps = steps
+        if denoise < 1.0:
+            total_steps = int(steps/denoise)
+
+        sigmas = get_sigmas_simple_kl_optimal(model.model, total_steps).cpu()
+        sigmas = sigmas[-(steps + 1):]
+        return (sigmas, )
+
+from .extra_samplers import get_sigmas_kl_optimal
+class KLOptimalScheduler:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"model": ("MODEL",),
+                     "steps": ("INT", {"default": 20, "min": 1, "max": 10000}),
+                     "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),
+                      }
+               }
+    RETURN_TYPES = ("SIGMAS",)
+    CATEGORY = "sampling/custom_sampling/schedulers"
+
+    FUNCTION = "get_sigmas"
+
+    def get_sigmas(self, model, steps, denoise):
+        total_steps = steps
+        if denoise < 1.0:
+            total_steps = int(steps/denoise)
+
+        sigmas = get_sigmas_kl_optimal(model.model, total_steps).cpu()
+        sigmas = sigmas[-(steps + 1):]
+        return (sigmas, )
+
 ### KSampler Nodes
 
 from comfy import model_management
@@ -730,16 +1032,21 @@ class WarmupDecayCFGGuider:
         return (guider,)
 
 class Guider_MegaCFG(comfy.samplers.CFGGuider):
-    def set_cfg(self, model, cfg_max, cfg_min, warmup_percent, mean_cfg):
+    def set_cfg(self, model, cfg_max, cfg_min, warmup_percent, mean_cfg, vector_rejection_scale):
         self.model = model
         self.cfg_max = cfg_max
         self.cfg_min = cfg_min
         self.warmup_percent = warmup_percent
         self.mean_cfg = mean_cfg
 
+        self.vector_rejection_scale = vector_rejection_scale
+
         self.prev_cond = None
         self.prev_cfg = None
         
+        self.cond_result = None
+        self.cfg_result = None
+
     def set_conds(self, positive, negative):
         self.inner_set_conds({"positive": positive, "negative": negative})
 
@@ -749,6 +1056,17 @@ class Guider_MegaCFG(comfy.samplers.CFGGuider):
         self.weight_scaling = weight_scaling
         self.latent_image = latent_image
     
+    def set_perphist_params(self, perphist):
+        self.perphist = perphist
+    
+    def perpadd(self, denoised_tensor, old_denoised_tensor, x, alpha):
+        a_diff = x - (denoised_tensor - x)
+        b_diff = x - (old_denoised_tensor - x)
+        a_ortho = a_diff * (a_diff / torch.linalg.norm(a_diff) * (b_diff / torch.linalg.norm(a_diff))).sum()
+        b_perp = b_diff - a_ortho
+        res = denoised_tensor + alpha * b_perp
+        return res
+
     def post_cfg_reference_img(self, args):
         model = args["model"]
         cond_pred = args["cond_denoised"]
@@ -777,6 +1095,24 @@ class Guider_MegaCFG(comfy.samplers.CFGGuider):
 
         return cfg_result + (cond_pred - ref) * self.image_guidance * (weight**self.weight_scaling)
     
+    def post_cfg_perphist(self, args):
+        noise_pred = args["denoised"]
+        if self.prev_cfg != None:
+            noise_pred = self.perpadd(noise_pred, self.prev_cfg, args["input"], self.perphist)
+        self.prev_cfg = args["denoised"]
+        return noise_pred
+    
+    def vect_rej(self, conditioning, unconditioning, x_input):
+        def rej(a, b):
+            """
+            Implements vector rejection for alternative diffusion.
+            """
+            return a - b * torch.tensordot(a, b, dims=4) / torch.tensordot(b, b, dims=4)
+        cond_ind_pred = conditioning - x_input
+        neg_ind_pred = unconditioning - x_input
+        noise_pred = rej(cond_ind_pred, neg_ind_pred) + (rej(x_input, cond_ind_pred) - rej(x_input, neg_ind_pred))
+        return noise_pred
+
     def predict_noise(self, x, timestep, model_options={}, seed=None):
         negative_cond = self.conds.get("negative", None)
         positive_cond = self.conds.get("positive", None)
@@ -802,12 +1138,14 @@ class Guider_MegaCFG(comfy.samplers.CFGGuider):
             cfg_cos = (1 + -torch.cos((timestep / percent_sigma) * math.pi))
             mod_cfg = cfg_scale * cfg_cos + self.cfg_min
 
+        if self.vector_rejection_scale != 0:
+            out[1] += self.vect_rej(out[1], out[0], x) * self.vector_rejection_scale
+
         cfg = comfy.samplers.cfg_function(self.inner_model, out[1], out[0], mod_cfg, x, timestep, model_options=model_options, cond=positive_cond, uncond=negative_cond)
 
         if self.mean_cfg != 0:
             cfg += out0_mean + (out1_mean - out0_mean) * self.mean_cfg
 
-        self.prev_cfg = cfg
         self.prev_cond = out[1]
 
         return cfg
@@ -823,6 +1161,8 @@ class MegaCFGGuider:
                     "cfg_min": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
                     "warmup_percent": ("FLOAT", {"default": 0.5, "min": 0.01, "max": 1.0, "step":0.01, "round": 0.001}),
                     "mean_cfg": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+                    "perphist": ("FLOAT", {"default": 0.0, "min": -1.0, "max": 1.0, "step":0.01, "round": 0.001}),
+                    "vector_rejection_scale": ("FLOAT", {"default": 0.0, "min": -0.5, "max": 5.0, "step":0.001, "round": 0.0001}),
                      },
                 "optional":
                     {
@@ -838,13 +1178,110 @@ class MegaCFGGuider:
     FUNCTION = "get_guider"
     CATEGORY = "sampling/custom_sampling/guiders"
 
-    def get_guider(self, model, positive, negative, cfg_max, cfg_min, warmup_percent, mean_cfg,
+    def get_guider(self, model, positive, negative, cfg_max, cfg_min, warmup_percent, mean_cfg, perphist, vector_rejection_scale,
                     image_guidance, image_weighting, weight_scaling, latent_image = None):
-        m = model.clone()
+        copy_model = perphist != 0 or latent_image != 0
+        m = model.clone() if copy_model else model
         guider = Guider_MegaCFG(m)
         guider.set_conds(positive, negative) # Conds
-        guider.set_cfg(m, cfg_max, cfg_min, warmup_percent, mean_cfg) # Strengths
+        guider.set_cfg(m, cfg_max, cfg_min, warmup_percent, mean_cfg, vector_rejection_scale) # Strengths
         if latent_image != None:
             guider.set_img_cfg(image_guidance, image_weighting, weight_scaling, latent_image)
             m.set_model_sampler_post_cfg_function(guider.post_cfg_reference_img)
+        if perphist != 0:
+            guider.set_perphist_params(perphist)
+            m.set_model_sampler_post_cfg_function(guider.post_cfg_perphist)
+        return (guider,)
+
+class Guider_APG(comfy.samplers.CFGGuider):    
+    class MomentumBuffer:
+        def __init__(self, momentum: float):
+            self.momentum = momentum
+            self.running_average = 0
+        def reset(self):
+            self.running_average = 0
+        def update(self, update_value: torch.Tensor):
+            new_average = self.momentum * self.running_average
+            self.running_average = update_value + new_average
+    
+    def project(self, v0: torch.Tensor, v1: torch.Tensor):
+        dtype = v0.dtype
+        v0, v1 = v0.double(), v1.double()
+        v1 = torch.nn.functional.normalize(v1, dim=[-3, -2, -1])
+        v0_parallel = (v0 * v1).sum(dim=[-3, -2, -1], keepdim=True) * v1
+        v0_orthogonal = v0 - v0_parallel
+        return v0_parallel.to(dtype), v0_orthogonal.to(dtype)
+
+    def set_cfg(self, cfg_scale, apg_scale, eta, norm_threshold, momentum_buffer):
+        self.cfg_scale = cfg_scale
+        self.apg_scale = apg_scale
+        self.eta = eta
+        self.norm_threshold = norm_threshold
+        self.momentum_buffer = momentum_buffer
+        
+        self.curr_timestep = 1.
+
+    def set_conds(self, positive, negative):
+        self.inner_set_conds({"positive": positive, "negative": negative})
+
+    def normalized_guidance(self, pred_cond: torch.Tensor, pred_uncond: torch.Tensor, guidance_scale: float, cfg: torch.Tensor = None, momentum_buffer: MomentumBuffer = None, eta: float = 1.0, norm_threshold: float = 0.0):
+        diff = pred_cond - pred_uncond
+
+        if momentum_buffer is not None:
+            momentum_buffer.update(diff)
+            diff = momentum_buffer.running_average
+        if norm_threshold > 0:
+            ones = torch.ones_like(diff)
+            diff_norm = diff.norm(p=2, dim=[-3, -2, -1], keepdim=True)
+            scale_factor = torch.minimum(ones, norm_threshold / diff_norm)
+            diff = diff * scale_factor
+        diff_parallel, diff_orthogonal = self.project(diff, pred_cond)
+        normalized_update = diff_orthogonal + eta * diff_parallel
+        if cfg is None:
+            cfg = pred_cond
+        pred_guided = cfg + (guidance_scale - 1) * normalized_update
+        return pred_guided
+
+    def predict_noise(self, x, timestep, model_options={}, seed=None):
+        negative = self.conds.get("negative", None)
+        positive_cond = self.conds.get("positive", None)
+
+        # Weird way to workaround momentum buffer sticking from run to run, this should automatically reset it in a majority of cases.
+        if timestep > (self.curr_timestep - 1e-6):
+            self.momentum_buffer.reset()
+        self.curr_timestep = timestep
+
+        out = comfy.samplers.calc_cond_batch(self.inner_model, [negative, positive_cond], x, timestep, model_options)
+
+        cfg = comfy.samplers.cfg_function(self.inner_model, out[1], out[0], self.cfg_scale, x, timestep, model_options=model_options, cond=positive_cond, uncond=negative)
+
+        apg = self.normalized_guidance(out[1], out[0], self.apg_scale, cfg, self.momentum_buffer, self.eta, self.norm_threshold)
+
+        return apg
+
+class APGGuider:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"model": ("MODEL",),
+                    "positive": ("CONDITIONING", ),
+                    "negative": ("CONDITIONING", ),
+                    "cfg_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+                    "apg_scale": ("FLOAT", {"default": 4.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+                    "eta": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+                    "norm_threshold": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step":0.1, "round": 0.01}),
+                    "momentum": ("FLOAT", {"default": -0.5, "min": -100.0, "max": 100.0, "step":0.1, "round": 0.01, "lazy": False}),
+                     }
+                }
+
+    RETURN_TYPES = ("GUIDER",)
+
+    FUNCTION = "get_guider"
+    CATEGORY = "sampling/custom_sampling/guiders"
+
+    def get_guider(self, model, positive, negative, cfg_scale, apg_scale, eta, norm_threshold, momentum):
+        guider = Guider_APG(model)
+        guider.set_conds(positive, negative) # Conds
+        momentum_buffer = guider.MomentumBuffer(momentum)
+        guider.set_cfg(cfg_scale, apg_scale, eta, norm_threshold, momentum_buffer) # Strengths
         return (guider,)