diff --git a/README.md b/README.md index 87069a7..bc3c7cc 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ This is the official implementation of ["Lanpaint: Training-Free Diffusion Inpai - **No Workarounds** – Generates 100% new content (no blending or smoothing) without relying on partial denoising. - **Beyond Inpainting** – You can even use it as a simple way to generate consistent characters. +**Warning**: LanPaint has degraded performance on distillation models, such as Flux.dev, due to a similar [issue with LORA training](https://medium.com/@zhiwangshi28/why-flux-lora-so-hard-to-train-and-how-to-overcome-it-a0c70bc59eaf). Please use low flux guidance (1.0-2.0) to mitigate this [issue](https://github.com/scraed/LanPaint/issues/30). + ## How It Works LanPaint uses Langevin Dynamics as "thinking" steps, which digs deeper into the diffusion process and allows the model to generate more consistent results. @@ -43,6 +45,9 @@ Once installed, you'll find the LanPaint nodes under the "sampling" category in ## Updates +- 2025/06/21 + - Update the algorithm with enhanced stability and outpaint performance. + - Add outpaint example - 2025/06/04 - Add more sampler support. - Add early stopping to advanced sampler. @@ -67,6 +72,12 @@ All examples use a random seed 0 to generate batch of 4 images for fair comparis You need to follow the ComfyUI version of [HiDream workflow](https://docs.comfy.org/tutorials/image/hidream/hidream-i1) to download and install the model. +### Example HiDream: OutPaint(LanPaint K Sampler, 5 steps of thinking) +![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_13(1).jpg) +[View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_10) + +You need to follow the ComfyUI version of [HiDream workflow](https://docs.comfy.org/tutorials/image/hidream/hidream-i1) to download and install the model. + ### Example SD 3.5: InPaint(LanPaint K Sampler, 5 steps of thinking) ![Inpainting Result 8](https://github.com/scraed/LanPaint/blob/master/examples/InpaintChara_12.jpg) [View Workflow & Masks](https://github.com/scraed/LanPaint/tree/master/examples/Example_9) diff --git a/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png b/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png index 112d86a..2aea28f 100644 Binary files a/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png and b/examples/Example_10/InPainted_Drag_Me_to_ComfyUI.png differ diff --git a/examples/InpaintChara_13(1).jpg b/examples/InpaintChara_13(1).jpg new file mode 100644 index 0000000..620dffb Binary files /dev/null and b/examples/InpaintChara_13(1).jpg differ diff --git a/src/LanPaint/lanpaint.py b/src/LanPaint/lanpaint.py index b54b69c..daa37c6 100644 --- a/src/LanPaint/lanpaint.py +++ b/src/LanPaint/lanpaint.py @@ -90,18 +90,37 @@ class LanPaint(): Gamma = Gamma_x * (1-mask) + Gamma_y * mask - + def Coef_C(x_t): + x0 = self.x0_evalutation(x_t, score, sigma, args) + C = (abt**0.5 * x0 - x_t )/ (1-abt) + A * x_t + return C + def advance_time(x_t, v, dt, Gamma, A, C, D): + dtype = x_t.dtype + with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): + osc = StochasticHarmonicOscillator(Gamma, A, C, D ) + x_t, v = osc.dynamics(x_t, v, dt ) + x_t = x_t.to(dtype) + v = v.to(dtype) + return x_t, v if args is None: #v = torch.zeros_like(x_t) v = None + C = Coef_C(x_t) + #print(torch.squeeze(dtx), torch.squeeze(dty)) + x_t, v = advance_time(x_t, v, dt, Gamma, A, C, D) else: - v, = args + v, C = args - with torch.autocast(device_type=x_t.device.type, dtype=torch.float32): - osc = StochasticHarmonicOscillator(Gamma, A, C, D ) - x_t, v = osc.dynamics(x_t, v, dt ) + x_t, v = advance_time(x_t, v, dt/2, Gamma, A, C, D) + + C_new = Coef_C(x_t) + v = v + Gamma**0.5 * ( C_new - C) *dt + + x_t, v = advance_time(x_t, v, dt/2, Gamma, A, C, D) + + C = C_new - return x_t, (v,) + return x_t, (v, C) def prepare_step_size(self, current_times, step_size, sigma_x, sigma_y): # ------------------------------------------------------------------------- @@ -125,7 +144,7 @@ class LanPaint(): Gamma_hat_y /= 2. A_t_x = (1) / ( 1 - abt ) * dtx / 2 - A_t_y = (1) / ( 1 - abt ) * dty / 2 + A_t_y = (1+self.chara_lamb) / ( 1 - abt ) * dty / 2 A_x = A_t_x / (dtx/2) A_y = A_t_y / (dty/2) diff --git a/src/LanPaint/nodes.py b/src/LanPaint/nodes.py index 5af99e3..b312fe8 100644 --- a/src/LanPaint/nodes.py +++ b/src/LanPaint/nodes.py @@ -260,7 +260,7 @@ class LanPaint_KSampler(): model.LanPaint_StepSize = 0.15 model.LanPaint_Lambda = 16.0 - model.LanPaint_Beta = 0.5 + model.LanPaint_Beta = 1. model.LanPaint_NumSteps = LanPaint_NumSteps model.LanPaint_Friction = 15. model.LanPaint_EarlyStop = 1 @@ -288,7 +288,7 @@ class LanPaint_KSamplerAdvanced: "end_at_step": ("INT", {"default": 10000, "min": 0, "max": 10000}), "return_with_leftover_noise": (["disable", "enable"], ), "LanPaint_NumSteps": ("INT", {"default": 5, "min": 0, "max": 100, "tooltip": "The number of steps for the Langevin dynamics, representing the turns of thinking per step."}), - "LanPaint_Lambda": ("FLOAT", {"default": 8., "min": 0.1, "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The bidirectional guidance scale. Higher values align with known regions more closely, but may result in instability."}), + "LanPaint_Lambda": ("FLOAT", {"default": 16., "min": 0.1, "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The bidirectional guidance scale. Higher values align with known regions more closely, but may result in instability."}), "LanPaint_StepSize": ("FLOAT", {"default": 0.15, "min": 0.0001, "max": 1., "step": 0.01, "round": 0.001, "tooltip": "The step size for the Langevin dynamics. Higher values result in faster convergence but may be unstable."}), "LanPaint_Beta": ("FLOAT", {"default": 1., "min": 0.0001, "max": 5, "step": 0.1, "round": 0.1, "tooltip": "The step size ratio between masked / unmasked regions. Lower value can compensate high values of LanPaint_Lambda."}), "LanPaint_Friction": ("FLOAT", {"default": 15, "min": 0., "max": 50.0, "step": 0.1, "round": 0.1, "tooltip": "The friction parameter for fast langevin, lower values result in faster convergence but may be unstable."}), diff --git a/src/LanPaint/utils.py b/src/LanPaint/utils.py index ad2f367..c017ab0 100644 --- a/src/LanPaint/utils.py +++ b/src/LanPaint/utils.py @@ -240,7 +240,6 @@ class StochasticHarmonicOscillator: tuple: (y(t), v(t)) """ - dummyzero = y0.new_zeros(1) # convert scalar to tensor with same device and dtype as y0 Delta = self.Delta + dummyzero Gamma_hat = self.Gamma * t + dummyzero @@ -253,8 +252,8 @@ class StochasticHarmonicOscillator: EE = 1 - Gamma_hat * zeta_2 if v0 is None: - #v0 = torch.randn_like(y0) * D / 2 ** 0.5 - v0 = (C - A * y0)/Gamma**0.5 + v0 = torch.randn_like(y0) * D / 2 ** 0.5 + #v0 = (C - A * y0)/Gamma**0.5 # Calculate mean position and velocity term1 = (1 - zeta_1) * (C * t - A * t * y0) + zeta_2 * (Gamma ** 0.5) * v0 * t @@ -273,16 +272,30 @@ class StochasticHarmonicOscillator: cov_matrix[..., 0, 1] = cov_yv cov_matrix[..., 1, 0] = cov_yv # symmetric cov_matrix[..., 1, 1] = cov_vv + + + # Compute the Cholesky decomposition to get scale_tril + #scale_tril = torch.linalg.cholesky(cov_matrix) + scale_tril = torch.zeros(*batch_shape, 2, 2, device=y0.device, dtype=y0.dtype) + tol = 1e-8 + cov_yy = torch.clamp( cov_yy, min = tol ) + sd_yy = torch.sqrt( cov_yy ) + inv_sd_yy = 1/(sd_yy) + + scale_tril[..., 0, 0] = sd_yy + scale_tril[..., 0, 1] = 0. + scale_tril[..., 1, 0] = cov_yv * inv_sd_yy + scale_tril[..., 1, 1] = torch.clamp( cov_vv - cov_yv**2 / cov_yy, min = tol ) ** 0.5 + # check if it matches torch.linalg. + #assert torch.allclose(torch.linalg.cholesky(cov_matrix), scale_tril, atol = 1e-4, rtol = 1e-4 ) # Sample correlated noise from multivariate normal mean = torch.zeros(*batch_shape, 2, device=y0.device, dtype=y0.dtype) mean[..., 0] = y_mean mean[..., 1] = v_mean new_yv = torch.distributions.MultivariateNormal( loc=mean, - covariance_matrix=cov_matrix + scale_tril=scale_tril ).sample() - return new_yv[...,0], new_yv[...,1] - - + return new_yv[...,0], new_yv[...,1] \ No newline at end of file