Fix FunPackAdvisorLLM: pass attention_mask to model.generate
When pad_token == eos_token the model cannot infer the attention mask from input_ids, so it warns and attends to all positions including padding. In practice this causes the model to echo the input prompt unchanged, randomly drop spaces, or produce garbled output depending on sampling temperature. Passing an all-ones attention_mask explicitly tells the model exactly which positions are real input tokens. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
27637d99df
commit
4c871b2d24
@@ -11207,6 +11207,10 @@ class _FunPackAdvisorLLMWrapper:
|
||||
# When thinking mode is active, add extra budget for the reasoning chain so the
|
||||
# actual response is not crowded out by thinking tokens.
|
||||
effective_max = int(max_length) + 2048 if thinking else int(max_length)
|
||||
# Explicit attention mask prevents the "pad token == eos token" warning and
|
||||
# stops the model from attending to padding positions, which causes it to echo
|
||||
# the input, drop spaces, or otherwise produce garbled output.
|
||||
attention_mask = torch.ones_like(input_ids)
|
||||
required = dict(
|
||||
do_sample=do_sample,
|
||||
max_new_tokens=effective_max,
|
||||
@@ -11215,6 +11219,7 @@ class _FunPackAdvisorLLMWrapper:
|
||||
top_p=float(top_p),
|
||||
repetition_penalty=float(repetition_penalty),
|
||||
pad_token_id=self._tokenizer.pad_token_id or self._tokenizer.eos_token_id,
|
||||
attention_mask=attention_mask,
|
||||
)
|
||||
# Optional params not supported by all models - try progressively stripping
|
||||
optional = {}
|
||||
|
||||
Reference in New Issue
Block a user