diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d98e68..67ec5bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Added an optional CLIP text-generation advisor to `FunPack Video Refiner V2`. The advisor uses an explicit Refiner system prompt, can run Gemma thinking mode, passes source images to compatible CLIP tokenizers, sees the prompt that caused the current rating, and can either report diagnostics or apply a validated prompt repair before encoding. +Added an optional `advisor_clip` input to `FunPack Video Refiner V2` so the advisor can use a separate generative CLIP/Gemma model while the main `clip` continues handling normal prompt encoding. + ## [2.4.2] - 2026-05-15 ### Added diff --git a/conditioning.py b/conditioning.py index 63be98f6..2840a363 100644 --- a/conditioning.py +++ b/conditioning.py @@ -5731,6 +5731,9 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): }, "optional": { "clip": ("CLIP", {"tooltip": "Optional text encoder. When connected, V2 encodes the prompt itself."}), + "advisor_clip": ("CLIP", { + "tooltip": "Optional separate CLIP/Gemma text generator for Advisor. If disconnected, Advisor falls back to the main CLIP.", + }), "positive_conditioning": ("CONDITIONING", { "tooltip": "Optional pre-encoded Gemma3/LTX2 conditioning. Used only when CLIP is not connected.", }), @@ -9562,7 +9565,8 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): def refine_v2(self, positive_prompt, clip=None, rating="Missing action", refinement_key="", seed=0, reset_session=False, lora_stack=None, im_feeling_lucky=False, user_intent_prompt="", refinement_key_input="", positive_conditioning=None, clip_vision_output=None, - source_image=None, negative_prompt="", mode="Refine", advisor_mode="Off", advisor_thinking=True): + source_image=None, negative_prompt="", mode="Refine", advisor_mode="Off", advisor_thinking=True, + advisor_clip=None): if seed != 0: torch.manual_seed(seed) random.seed(seed) @@ -9577,6 +9581,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): execution_mode = self._v2_execution_mode(mode) learning_mode = execution_mode == "Learning" advisor_mode = self._v2_advisor_mode(advisor_mode) + advisor_clip = advisor_clip if advisor_clip is not None else clip state, state_status = self._v2_load_state(refinement_key, reset_session=reset_session) scene_db, scene_builder_status = self._v2_scene_builder_db(state) global_state = state.setdefault("global", {}) @@ -9780,7 +9785,7 @@ class FunPackVideoRefinerV2(FunPackVideoRefiner): if not current_prompt_refusal: prompt_to_encode, advisor_status, advisor_diagnostic, advisor_applied = self._v2_prompt_advisor( - clip, + advisor_clip, advisor_mode, prompt_to_encode, intent_source_prompt, diff --git a/docs/FunPackVideoRefinerV2.md b/docs/FunPackVideoRefinerV2.md index a73a02af..8277bbcb 100644 --- a/docs/FunPackVideoRefinerV2.md +++ b/docs/FunPackVideoRefinerV2.md @@ -10,6 +10,8 @@ V2 replaces the old `FunPack Video Refiner` public node in version `2.2.0`. It u **clip**: Connected text encoder. V2 uses this CLIP for prompt encoding, phrase/category similarity checks, and optional advisor text generation when the connected CLIP supports `generate`/`decode`. +**advisor_clip**: Optional separate CLIP/Gemma text generator for the Advisor. When connected, Advisor uses this model for text generation while the main `clip` still owns prompt encoding and similarity checks. When disconnected, Advisor falls back to `clip`. If neither selected CLIP exposes text generation, Advisor skips and V2 continues with the usual workflow. + **rating**: Feedback for the previous V2 generation: - `Perfect` diff --git a/tests/test_v2_category_alignment.py b/tests/test_v2_category_alignment.py index 9f296d7d..78638f8c 100644 --- a/tests/test_v2_category_alignment.py +++ b/tests/test_v2_category_alignment.py @@ -1456,6 +1456,7 @@ def test_refiner_v2_exposes_clip_and_conditioning_as_optional_inputs(): assert inputs["required"]["mode"][0] == ["Refine", "Learning"] assert inputs["required"]["advisor_mode"][0] == ["Off", "Diagnostics", "Repair prompt"] assert "clip" in inputs["optional"] + assert "advisor_clip" in inputs["optional"] assert "positive_conditioning" in inputs["optional"] assert inputs["optional"]["advisor_thinking"][1]["default"] is True @@ -1560,6 +1561,126 @@ def test_refiner_v2_advisor_repair_applies_validated_generated_prompt(tmp_path): assert state["last_run"]["advisor"]["applied"] is True +def test_refiner_v2_advisor_uses_separate_advisor_clip_when_connected(tmp_path): + refiner = FunPackVideoRefinerV2() + state_path = tmp_path / "state.json" + state_path.write_text(json.dumps({ + "version": 2, + "refinement_key": "advisor-clip-test", + "state_namespace": "clip", + "global": { + "total_iterations": 1, + "avg_reward_ema": 0.0, + "good_streak": 0, + "bad_streak": 1, + "last_rating_label": "Missing action", + "last_missing_axes": ["action"], + "phrase_memory": {}, + "axis_conditioning_memory": {}, + "lora_weight_memory": {}, + "preferred_context_memory": {}, + "intent_alignment_memory": {}, + "intent_family_memory": {}, + "perfect_anchors": {}, + "variant_evidence": {}, + "intent_preference_phrases": {}, + "conditioning_deltas": {}, + "active_repair_axes": [], + "negative_prompt_memory": {}, + "vision_memory": {}, + "loss_history": [], + }, + "prompt_histories": {}, + "last_run": { + "prompt": "person smoking", + "encoded_prompt": "person smoking", + "source_conditioning": tensor_to_serializable(torch.zeros(1, 4, 3)), + "conditioning": tensor_to_serializable(torch.zeros(1, 4, 3)), + "phrases": prompt_phrases(refiner, "person smoking"), + "rating_label": "Unrated", + "iteration": 1, + }, + }), encoding="utf-8") + refiner._v2_state_path = lambda refinement_key: str(state_path) + main_clip = CountingClip() + advisor_clip = GeneratingClip( + "DIAGNOSTIC: add visible smoke motion.\n" + "REPAIRED_PROMPT: person smoking, smoke trails drifting upward" + ) + + refiner.refine_v2( + "person smoking", + main_clip, + "Missing details", + "advisor-clip-test", + user_intent_prompt="person smoking", + advisor_mode="Repair prompt", + advisor_clip=advisor_clip, + ) + + state = json.loads(state_path.read_text(encoding="utf-8")) + assert advisor_clip.tokenize_calls + assert state["last_run"]["encoded_prompt"] == "person smoking, smoke trails drifting upward" + assert state["last_run"]["advisor"]["applied"] is True + + +def test_refiner_v2_advisor_skips_when_no_generation_clip_is_available(tmp_path): + refiner = FunPackVideoRefinerV2() + state_path = tmp_path / "state.json" + state_path.write_text(json.dumps({ + "version": 2, + "refinement_key": "advisor-no-generation-test", + "state_namespace": "clip", + "global": { + "total_iterations": 1, + "avg_reward_ema": 0.0, + "good_streak": 0, + "bad_streak": 1, + "last_rating_label": "Missing action", + "last_missing_axes": ["action"], + "phrase_memory": {}, + "axis_conditioning_memory": {}, + "lora_weight_memory": {}, + "preferred_context_memory": {}, + "intent_alignment_memory": {}, + "intent_family_memory": {}, + "perfect_anchors": {}, + "variant_evidence": {}, + "intent_preference_phrases": {}, + "conditioning_deltas": {}, + "active_repair_axes": [], + "negative_prompt_memory": {}, + "vision_memory": {}, + "loss_history": [], + }, + "prompt_histories": {}, + "last_run": { + "prompt": "person smoking", + "encoded_prompt": "person smoking", + "source_conditioning": tensor_to_serializable(torch.zeros(1, 4, 3)), + "conditioning": tensor_to_serializable(torch.zeros(1, 4, 3)), + "phrases": prompt_phrases(refiner, "person smoking"), + "rating_label": "Unrated", + "iteration": 1, + }, + }), encoding="utf-8") + refiner._v2_state_path = lambda refinement_key: str(state_path) + + _, status, training_info, _, _ = refiner.refine_v2( + "person smoking", + FakeClip(), + "Missing details", + "advisor-no-generation-test", + advisor_mode="Repair prompt", + ) + + state = json.loads(state_path.read_text(encoding="utf-8")) + assert "Advisor: unavailable; connected CLIP does not expose text generation" in status + assert "Advisor: unavailable; connected CLIP does not expose text generation" in training_info + assert state["last_run"]["encoded_prompt"] == "person smoking" + assert state["last_run"]["advisor"]["applied"] is False + + def test_refiner_v2_learning_mode_passes_prompt_and_conditioning_through(tmp_path): refiner = FunPackVideoRefinerV2() state_path = tmp_path / "state.json"