From 00cd460e3c92ad3e3a392e2f42513ca29d461966 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Sat, 30 May 2026 12:54:41 +0300 Subject: [PATCH] Fix value function reward hacking: cap conditioning ascent displacement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unconstrained ascent was stripping action content from conditioning to reach a "safe" static-character attractor. Added 8% displacement cap relative to original conditioning norm — quality direction is still steered but the prompt content can't be erased. Also raised MIN_SAMPLES from 5 to 10 to reduce overfitting risk before the value function activates. Co-Authored-By: Claude Sonnet 4.6 --- value_function.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/value_function.py b/value_function.py index 752bb475..368a8ac4 100644 --- a/value_function.py +++ b/value_function.py @@ -12,7 +12,7 @@ import torch.nn.functional as F class OnlineValueFunction(nn.Module): - MIN_SAMPLES = 5 + MIN_SAMPLES = 10 BUFFER_SIZE = 100 BATCH_SIZE = 16 TRAIN_STEPS = 20 @@ -93,10 +93,13 @@ class OnlineValueFunction(nn.Module): orig_device = conditioning.device orig_dtype = conditioning.dtype step_size = conditioning.float().norm().item() * 0.005 + # Cap total displacement at 8% of original norm — prevents stripping action/content + max_displacement = conditioning.float().norm().item() * 0.08 prev_reward = -float("inf") with torch.inference_mode(False), torch.enable_grad(): - c = torch.empty(conditioning.shape, dtype=torch.float32, device=mlp_device) - c.copy_(conditioning) + c_orig = torch.empty(conditioning.shape, dtype=torch.float32, device=mlp_device) + c_orig.copy_(conditioning) + c = c_orig.clone() for _ in range(50): c = c.detach().requires_grad_(True) reward = self.forward(self.compress(c).unsqueeze(0)) @@ -104,6 +107,11 @@ class OnlineValueFunction(nn.Module): reward_val = reward.item() grad = F.normalize(c.grad.float(), dim=-1) c = c.detach() + step_size * grad + # Project back if displacement exceeds cap + delta = c - c_orig + disp = delta.norm() + if disp > max_displacement: + c = c_orig + delta * (max_displacement / disp) if reward_val - prev_reward < 1e-3: break prev_reward = reward_val