diff --git a/libs/Ditto/core/atomic_components/audio2motion.py b/libs/Ditto/core/atomic_components/audio2motion.py index af3bdee..e3355c0 100644 --- a/libs/Ditto/core/atomic_components/audio2motion.py +++ b/libs/Ditto/core/atomic_components/audio2motion.py @@ -166,6 +166,18 @@ class Audio2Motion: return res_kp_seq + def _generate_gaussian_pitch(self, target_deg, sigma=1.5): + # Ditto Pitch: 66 bins, 3 deg each, center at 32.5 (0 deg) + # target_deg = (idx * 3) - 97.5 => idx = (target_deg + 97.5) / 3 + mu = (target_deg + 97.5) / 3.0 + x = np.arange(66) + # We generate raw coefficients that will simulate the target degree + gaussian = np.exp(-((x - mu)**2) / (2 * sigma**2)) + # Add tiny epsilon to avoid log(0) issues in model + gaussian = np.clip(gaussian, 1e-6, 1.0) + # Log-space coefficients for the categorical model + return np.log(gaussian).reshape(1, 66) + def _update_kp_cond(self, res_kp_seq, idx, step_len=0): # [v1.9.144] Unconditional State Tracking # We process these outside the fix_kp_cond branches to ensure monitoring never stops. @@ -205,39 +217,45 @@ class Audio2Motion: if self.is_recovering: self.look_up_timer += step_len - # [v1.9.162] PREDICTIVE POSTURE STATE MACHINE - # We scan the VAD timeline to decide the character's "Readiness" + # [v1.9.165] FLUID POSTURE STATE MACHINE + # Decisions are based on Silence Frames (Universal) + VAD Lookahead (Optional) target_bias_deg = 0.0 # Standard flat + is_currently_talking = self.silence_frames < 25 + has_upcoming_speech = False + if self.vad_timeline is not None: - # Lookahead window: 2 seconds (50 frames) lookahead = self.vad_timeline[idx : idx + 50] - - is_currently_talking = self.silence_frames < 25 has_upcoming_speech = len(lookahead) > 0 and np.max(lookahead) > 0.1 - - if is_currently_talking or has_upcoming_speech: - # COMBAT STANCE: Tucked chin (-5 deg downward = -0.0020 impulse push) - target_bias_deg = +5.0 - self.target_push = -0.0025 # Persistent negative impulse to lower head - else: - # IDLE SWAY: Natural breathing motion - cycle = np.sin(self.global_time * 0.05) - target_bias_deg = -3.0 + cycle * 5.0 - self.target_push = 0.0 # No forced push, just natural sway + + if is_currently_talking or has_upcoming_speech: + # COMBAT STANCE: Tucked chin for headroom + target_bias_deg = -5.0 else: - self.target_push = 0.0 - target_bias_deg = 0.0 - - # Smooth Push Transition - self.current_push = self.current_push * 0.90 + self.target_push * 0.10 + # IDLE SWAY: Natural breathing + cycle = np.sin(self.global_time * 0.05) + target_bias_deg = +2.0 + cycle * 5.0 # Slightly down/neutral sway + + # [v1.9.165] Synthetic Anchor Generation + # Instead of photo bias, we anchor to a mathematical IDEAL distribution. self.target_bias_deg = target_bias_deg + synthetic_pitch = self._generate_gaussian_pitch(target_bias_deg) + + # Apply to anchor target + self.target_neutralizer = self.photo_base_neutralizer.copy() + # Offset 1:67 is Pitch. We overwrite with synthetic distribution. + # Since photo_base_neutralizer[1:67] is -s_kp[1:67], + # photo_base + s_kp + synthetic = synthetic. + self.target_neutralizer[0, 1:67] = -self.s_kp_cond[0, 1:67] + synthetic_pitch[0] + + # Smooth Transition (0.05 EMA = ~1s glide) + self.current_neutralizer = self.current_neutralizer * 0.95 + self.target_neutralizer * 0.05 tag = "[HEARTBEAT]" if not self.is_recovering else "[RECOVERY]" - if self.vad_timeline is not None and has_upcoming_speech and not is_currently_talking: + if has_upcoming_speech and not is_currently_talking: tag = "[ANTICIPATION]" - print(f"{tag} Frame {idx:04d} | Delta={delta_p:+.2f}° | Bias={self.target_bias_deg:+.1f}° | Silence={self.silence_frames:03d}") + print(f"{tag} Frame {idx:04d} | Delta={delta_p:+.2f}° | Goal={self.target_bias_deg:+.1f}° | Silence={self.silence_frames:03d}") # 5. Postural Auto-Correction Logic [REPLACED by v1.9.160 STATIC + v1.9.162 PREDICTIVE] @@ -381,25 +399,26 @@ class Audio2Motion: pred_kp_seq = self.lmdm(self.kp_cond, aud_cond, self.sampling_timesteps) - # [v1.9.160/162/164] STATIC/PREDICTIVE POSTURE MODE - # We now enforce high Pitch pressure ALWAYS to anchor to our target baseline. + # [v1.9.165] FLUID POSTURE MODE + # We now use "Mathematical Gravity" to pull the head into the safe zone. if True: - # Constant 95% tension for Pitch (Vertical) - pressure = 0.95 - soft_p = 0.40 # Maintain soft-lock for Yaw/Roll to allow mouth projection + # [v1.9.165] Fluid Pressure: 75% Pitch / 30% Yaw-Roll + # This restores expressiveness while maintaining vertical stability. + pressure = 0.75 + soft_p = 0.30 - # [v1.9.164] Stable Anchor = Photo + Drift-Adjusted Offset - anchor_p = (self.s_kp_cond + self.brownian_pos)[0, 1:202] + # Stable Anchor = Photo + Synthetic Offset + Brownian Drift + anchor_p = (self.s_kp_cond + self.current_neutralizer + self.brownian_pos)[0, 1:202] for f in range(pred_kp_seq.shape[1]): # Absolute Pitch Locking (Halt baring distortion) pred_kp_seq[0, f, 1:67] = pred_kp_seq[0, f, 1:67] * (1.0 - pressure) + anchor_p[0:66] * pressure - # Soft Rotational Locking (Organic freedom) + # Soft Rotational Locking (Organic movement) pred_kp_seq[0, f, 67:202] = pred_kp_seq[0, f, 67:202] * (1.0 - soft_p) + anchor_p[66:201] * soft_p if self.clip_idx % 20 == 0: - mode_s = "SPEECH" if self.target_bias_deg > 2.0 else "IDLE" - print(f"[v1.9.162 {mode_s}] Anchor: {self.target_bias_deg:+.1f}° goal.") + mode_s = "SPEECH" if self.target_bias_deg < -2.0 else "IDLE" + print(f"[v1.9.165 {mode_s}] Fluid Gravity: {self.target_bias_deg:+.1f}° goal.") # [v1.9.156] Virtual Last Frame for Startup Stabilization # If this is the VERY first chunk, we treat the source photo as the "prev frame" diff --git a/pyproject.toml b/pyproject.toml index d01b5c6..65f618f 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "aiia" description = "The Ultimate AI Audio/Video toolkit for ComfyUI. Features an enhanced Ditto (with optimizations that outperform official demos and other SOTA talking head models in lip-sync accuracy and natural motion), EchoMimic V3 & FLOAT, VibeVoice & CosyVoice 3.0 (Zero-Shot Voice Cloning), Multi-Role Podcast Generation, and a powerful Media Browser." -version = "1.9.164" +version = "1.9.165" license = {file = "LICENSE"} readme = "README.md" authors = [