From 727f80bd09b5de4fac0b61ed6e25f270ee80b3b6 Mon Sep 17 00:00:00 2001 From: Austin Mroz Date: Tue, 19 Sep 2023 01:07:45 -0500 Subject: [PATCH 1/3] Possible fixes to uniform context window generator Prevent duplicate frames on highest context_stride Modify closed_loop to cause a window break instead of modularly wrapping --- animatediff/context.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/animatediff/context.py b/animatediff/context.py index 99ba794..45a0ce7 100644 --- a/animatediff/context.py +++ b/animatediff/context.py @@ -44,13 +44,33 @@ def uniform( context_stride = min(context_stride, int(np.ceil(np.log2(num_frames / context_size))) + 1) + pad = int(round(num_frames * ordered_halving(step, print_final))) for context_step in 1 << np.arange(context_stride): - pad = int(round(num_frames * ordered_halving(step, print_final))) + j_initial = int(ordered_halving(step) * context_step) + pad for j in range( - int(ordered_halving(step) * context_step) + pad, - num_frames + pad + (0 if closed_loop else -context_overlap), + j_initial, + num_frames + pad - context_overlap, (context_size * context_step - context_overlap), ): + if context_size * context_step > num_frames: + # On the final context_step, this prevents the window, + # ensure no frame appears in the window twice + yield [e % num_frames for e in range(j, j + num_frames, context_step)] + continue + j = j % num_frames + if j > (j + context_size * context_step) % num_frames and not closed_loop: + yield [e for e in range(j, num_frames, context_step)] + j_stop = (j + context_size * context_step) % num_frames + # When ((num_frames % (context_size - context_overlap)+context_overlap) % context_size != 0, + # This can cause 'superflous' runs where all frames in + # a context window have already been processed during + # the first context window of this stride and step. + # While the following commented if should prevent this, + # I believe leaving it in is more correct as it maintains + # the total conditional passes per frame over a large total steps + # if j_stop > context_overlap: + yield [e for e in range(0, j_stop, context_step)] + continue yield [e % num_frames for e in range(j, j + context_size * context_step, context_step)] From 26fd2aa977896abb15c7e6eb1a0fd53bc0fea918 Mon Sep 17 00:00:00 2001 From: Austin Mroz Date: Tue, 19 Sep 2023 01:38:47 -0500 Subject: [PATCH 2/3] Move potential fixes to a new uniform v2 sampler Names can be changed later, but this still allows for easier testing of what differences the proposed changes make. It also provides an excuse to fix a grammatically incorrect comment --- animatediff/context.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/animatediff/context.py b/animatediff/context.py index 45a0ce7..ef0d23b 100644 --- a/animatediff/context.py +++ b/animatediff/context.py @@ -7,8 +7,9 @@ import numpy as np class ContextSchedules: UNIFORM = "uniform" UNIFORM_CONSTANT = "uniform_constant" + UNIFORM_V2 = "uniform v2" - CONTEXT_SCHEDULE_LIST = [UNIFORM] + CONTEXT_SCHEDULE_LIST = [UNIFORM, UNIFORM_V2] # Returns fraction that has denominator that is a power of 2 @@ -44,6 +45,31 @@ def uniform( context_stride = min(context_stride, int(np.ceil(np.log2(num_frames / context_size))) + 1) + for context_step in 1 << np.arange(context_stride): + pad = int(round(num_frames * ordered_halving(step, print_final))) + for j in range( + int(ordered_halving(step) * context_step) + pad, + num_frames + pad + (0 if closed_loop else -context_overlap), + (context_size * context_step - context_overlap), + ): + yield [e % num_frames for e in range(j, j + context_size * context_step, context_step)] + +def uniform_v2( + step: int = ..., + num_steps: Optional[int] = None, + num_frames: int = ..., + context_size: Optional[int] = None, + context_stride: int = 3, + context_overlap: int = 4, + closed_loop: bool = True, + print_final: bool = False, +): + if num_frames <= context_size: + yield list(range(num_frames)) + return + + context_stride = min(context_stride, int(np.ceil(np.log2(num_frames / context_size))) + 1) + pad = int(round(num_frames * ordered_halving(step, print_final))) for context_step in 1 << np.arange(context_stride): j_initial = int(ordered_halving(step) * context_step) + pad @@ -53,7 +79,7 @@ def uniform( (context_size * context_step - context_overlap), ): if context_size * context_step > num_frames: - # On the final context_step, this prevents the window, + # On the final context_step, # ensure no frame appears in the window twice yield [e % num_frames for e in range(j, j + num_frames, context_step)] continue From 04ada0a89803f240b1a83e9d43d9fa3863738922 Mon Sep 17 00:00:00 2001 From: Austin Mroz Date: Tue, 19 Sep 2023 12:01:52 -0500 Subject: [PATCH 3/3] Actually add uniform_v2 method lookup --- animatediff/context.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/animatediff/context.py b/animatediff/context.py index ef0d23b..60b7fd7 100644 --- a/animatediff/context.py +++ b/animatediff/context.py @@ -149,6 +149,8 @@ def get_context_scheduler(name: str) -> Callable: return uniform case ContextSchedules.UNIFORM_CONSTANT: return uniform_constant + case ContextSchedules.UNIFORM_V2: + return uniform_v2 case _: raise ValueError(f"Unknown context_overlap policy {name}")