diff --git a/compiler/analyzer/backend.py b/compiler/analyzer/backend.py index 7459b5e..99f2c3d 100644 --- a/compiler/analyzer/backend.py +++ b/compiler/analyzer/backend.py @@ -73,7 +73,9 @@ class BackendResponseError(AnalyzerBackendError): class AnalyzerBackend(Protocol): """A language-model backend that turns a prompt into raw text.""" - def generate(self, prompt: str) -> BackendResult: # pragma: no cover - interface + def generate( + self, prompt: str, *, temperature: float | None = None + ) -> BackendResult: # pragma: no cover - interface ... @@ -118,20 +120,27 @@ class OllamaBackend: transport=transport, ) - def generate(self, prompt: str) -> BackendResult: + def generate(self, prompt: str, *, temperature: float | None = None) -> BackendResult: """Send ``prompt`` unmodified to Ollama and return the raw text response. + Args: + prompt: The full prompt, sent byte-for-byte unchanged. + temperature: Optional per-call sampling temperature; falls back to the + backend's configured temperature when ``None``. Used by the retry + orchestration to perturb sampling without altering the prompt. + Raises: BackendTimeoutError: On timeout (SC0013). BackendUnavailableError: On connection failure (SC0012). BackendResponseError: On an HTTP error or unexpected response (SC0018). """ url = f"{self._base_url}/api/generate" + effective_temperature = self._temperature if temperature is None else temperature payload = { "model": self._model, "prompt": prompt, "stream": False, - "options": {"temperature": self._temperature}, + "options": {"temperature": effective_temperature}, } start = time.perf_counter() diff --git a/compiler/analyzer/scene_analyzer.py b/compiler/analyzer/scene_analyzer.py index 56ee344..d5c25c5 100644 --- a/compiler/analyzer/scene_analyzer.py +++ b/compiler/analyzer/scene_analyzer.py @@ -27,6 +27,22 @@ from compiler.common.config import Config from compiler.common.log import StructuredLogger from compiler.common.result import CompilerResult +# Retry temperature escalation (ยง12.9). The prompt is reused byte-for-byte on +# every attempt, so at temperature 0 a bad-JSON attempt would deterministically +# repeat and every retry would fail identically. Escalating the sampling +# temperature on retries only (never the first attempt) lets a malformed run +# self-recover, exactly like a fresh manual re-run, while keeping the happy path +# deterministic. +_TEMPERATURE_STEP = 0.2 +_TEMPERATURE_CAP = 1.0 + + +def _attempt_temperature(base: float, attempt_index: int) -> float: + """Temperature for a 0-based attempt: the base first, then escalating on retries.""" + if attempt_index == 0: + return base + return min(base + _TEMPERATURE_STEP * attempt_index, _TEMPERATURE_CAP) + def analyze( description: str, @@ -59,10 +75,13 @@ def analyze( attempts = 0 raw_response = "" + base_temperature = config.analyzer.temperature + for _ in range(max_attempts): + temperature = _attempt_temperature(base_temperature, attempts) attempts += 1 try: - backend_result = backend.generate(prompt) + backend_result = backend.generate(prompt, temperature=temperature) except (BackendUnavailableError, BackendTimeoutError) as exc: # Terminal: the backend is unreachable or too slow; retrying cannot help. return _finalize( diff --git a/tests/analyzer/test_ollama_backend.py b/tests/analyzer/test_ollama_backend.py index 9b76391..88f26d7 100644 --- a/tests/analyzer/test_ollama_backend.py +++ b/tests/analyzer/test_ollama_backend.py @@ -61,6 +61,28 @@ def test_successful_generate_returns_text_and_telemetry() -> None: assert captured["url"].endswith("/api/generate") +def test_temperature_override_reaches_payload() -> None: + captured = {} + + def transport(url, payload, timeout): + captured["payload"] = payload + return {"response": "x"} + + make_backend(transport).generate("hi", temperature=0.4) + assert captured["payload"]["options"]["temperature"] == pytest.approx(0.4) + + +def test_temperature_defaults_to_configured_when_not_overridden() -> None: + captured = {} + + def transport(url, payload, timeout): + captured["payload"] = payload + return {"response": "x"} + + make_backend(transport).generate("hi") + assert captured["payload"]["options"]["temperature"] == pytest.approx(0.0) + + # --- failure modes --------------------------------------------------------- diff --git a/tests/analyzer/test_scene_analyzer.py b/tests/analyzer/test_scene_analyzer.py index a1024ac..d7c239f 100644 --- a/tests/analyzer/test_scene_analyzer.py +++ b/tests/analyzer/test_scene_analyzer.py @@ -4,6 +4,8 @@ from __future__ import annotations import json +import pytest + from compiler.analyzer.backend import ( BackendResult, BackendTimeoutError, @@ -46,10 +48,12 @@ class FakeBackend: self._script = list(script) self.calls = 0 self.prompts: list[str] = [] + self.temperatures: list[float | None] = [] - def generate(self, prompt: str) -> BackendResult: + def generate(self, prompt: str, *, temperature: float | None = None) -> BackendResult: self.calls += 1 self.prompts.append(prompt) + self.temperatures.append(temperature) item = self._script.pop(0) if isinstance(item, Exception): raise item @@ -88,6 +92,29 @@ def test_prompt_is_reused_unchanged_across_retries() -> None: assert len(set(backend.prompts)) == 1 # identical prompt every attempt +def test_temperature_escalates_on_retries_only() -> None: + # First attempt at the base temperature (deterministic), retries escalate so a + # bad-JSON run can self-recover without a manual re-run. + backend = FakeBackend(["not json", "still not json", valid_scene_text()]) + analyze("A girl.", backend, config()) + assert backend.temperatures == pytest.approx([0.0, 0.2, 0.4]) + + +def test_temperature_escalation_builds_on_configured_base() -> None: + backend = FakeBackend(["bad", valid_scene_text()]) + cfg = Config.from_json({"analyzer": {"temperature": 0.5}}) + analyze("A girl.", backend, cfg) + assert backend.temperatures == pytest.approx([0.5, 0.7]) + + +def test_temperature_is_capped() -> None: + backend = FakeBackend(["bad"] * 10) + cfg = Config.from_json({"analyzer": {"temperature": 0.9, "max_retries": 5}}) + analyze("A girl.", backend, cfg) + assert max(backend.temperatures) <= 1.0 + assert backend.temperatures[-1] == pytest.approx(1.0) # escalation is clamped at the cap + + def test_prompt_includes_system_prompt_and_description() -> None: backend = FakeBackend([valid_scene_text()]) analyze("A blonde girl.", backend, config())