From 061269a41f1d4dfdec37fc384df123f258c26f25 Mon Sep 17 00:00:00 2001 From: digital-garbage <133010426+digital-garbage@users.noreply.github.com> Date: Wed, 6 May 2026 23:07:09 +0300 Subject: [PATCH] Remember lucky phrase placement --- CHANGELOG.md | 2 + conditioning.py | 150 ++++++++++++++++++++++++++- docs/FunPackGemmaEmbeddingRefiner.md | 2 +- 3 files changed, 149 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85a5c3e1..cb4bf413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ Fixed `I'm Feeling Lucky` CLIP/Gemma re-encode crashes when the encoded Lucky pr Updated `I'm Feeling Lucky` CLIP/Gemma prompt composition to preserve learned comma/semicolon-separated concept phrases instead of emitting loose word lists when phrase memory is available. +Added Lucky phrase placement memory so learned prompt phrases remember their rated order positions and CLIP/Gemma Lucky prompts can reassemble phrases into a more coherent prompt order instead of sentence salad. + ## [2.1.3] - 2026-04-24 ### Changed diff --git a/conditioning.py b/conditioning.py index e7a56a3f..7021a418 100644 --- a/conditioning.py +++ b/conditioning.py @@ -856,6 +856,53 @@ class FunPackVideoRefiner: global_adaptive["lucky_context_memory_validated"] = True return cleaned + def _phrase_position_score(self, item): + likes = float(item.get("liked_count", 0)) + neutral = float(item.get("neutral_count", 0)) + awful = float(item.get("awful_count", 0)) + return float(item.get("score", 0.0)) + likes * 0.45 + neutral * 0.06 - awful * 0.85 + + def _ensure_lucky_phrase_placements(self, global_adaptive): + memory = global_adaptive.get("lucky_phrase_placements") + if not isinstance(memory, dict): + memory = {} + if memory and global_adaptive.get("lucky_phrase_placements_validated") is True: + return memory + + cleaned = {} + for phrase, item in memory.items(): + phrase_key = str(phrase).strip().lower() + if len(phrase_key) < 3 or not isinstance(item, dict): + continue + positions = item.get("positions") + if not isinstance(positions, dict): + continue + clean_positions = {} + for pos, pos_item in positions.items(): + if not isinstance(pos_item, dict): + continue + try: + pos_key = str(max(0, min(31, int(pos)))) + pos_item["score"] = round(max(-4.0, min(8.0, float(pos_item.get("score", 0.0)))), 4) + pos_item["liked_count"] = max(0, int(pos_item.get("liked_count", 0))) + pos_item["neutral_count"] = max(0, int(pos_item.get("neutral_count", 0))) + pos_item["awful_count"] = max(0, int(pos_item.get("awful_count", 0))) + pos_item["count"] = max(0, int(pos_item.get("count", 0))) + pos_item["last_seen_iter"] = max(0, int(pos_item.get("last_seen_iter", 0))) + clean_positions[pos_key] = pos_item + except (TypeError, ValueError): + continue + if not clean_positions: + continue + cleaned[phrase_key] = { + "text": str(item.get("text", phrase_key)).strip().lower() or phrase_key, + "positions": clean_positions, + } + + global_adaptive["lucky_phrase_placements"] = cleaned + global_adaptive["lucky_phrase_placements_validated"] = True + return cleaned + def _rating_memory_deltas(self, rating_key): if rating_key == "like": return 0.70, 1, 0, 0 @@ -908,6 +955,34 @@ class FunPackVideoRefiner: return memory + def _update_lucky_phrase_placements(self, global_adaptive, prompt_text, rating_key, iter_num): + phrases = self._ordered_prompt_phrases(prompt_text) + if not phrases: + return self._ensure_lucky_phrase_placements(global_adaptive) + + memory = self._ensure_lucky_phrase_placements(global_adaptive) + score_delta, liked_delta, neutral_delta, awful_delta = self._rating_memory_deltas(rating_key) + for index, phrase in enumerate(phrases[:32]): + text = str(phrase.get("text", "")).strip().lower() + if len(text) < 3: + continue + entry = memory.setdefault(text, {"text": text, "positions": {}}) + positions = entry.setdefault("positions", {}) + pos_key = str(index) + item = positions.get(pos_key, {}) + item["score"] = round(max(-4.0, min(8.0, float(item.get("score", 0.0)) + score_delta)), 4) + item["liked_count"] = max(0, int(item.get("liked_count", 0)) + liked_delta) + item["neutral_count"] = max(0, int(item.get("neutral_count", 0)) + neutral_delta) + item["awful_count"] = max(0, int(item.get("awful_count", 0)) + awful_delta) + item["count"] = max(0, int(item.get("count", 0)) + 1) + item["last_seen_iter"] = int(iter_num) + positions[pos_key] = item + entry["text"] = text + memory[text] = entry + + global_adaptive["lucky_phrase_placements"] = memory + return memory + def _update_void_token_pairs(self, global_adaptive, word_groups, rating_key, iter_num): pair_bank = self._ensure_void_pair_bank(global_adaptive) ordered_tokens = [] @@ -1469,6 +1544,7 @@ class FunPackVideoRefiner: chosen_set = set(prompt_tokens) if isinstance(phrase_sequences, list): phrase_candidates = [] + placement_memory = self._ensure_lucky_phrase_placements(global_adaptive) for sequence_index, sequence in enumerate(phrase_sequences[-64:]): if not isinstance(sequence, list): continue @@ -1491,16 +1567,37 @@ class FunPackVideoRefiner: score = sum(max(0.05, token_scores.get(token, 0.0) + 1.25) for token in overlap) score += 0.18 / float(phrase_index + 1) score += 0.03 / float(max(1, len(phrase_sequences) - sequence_index)) - phrase_candidates.append((score, text, tokens)) + preferred_position, placement_score = self._lucky_phrase_preferred_position( + placement_memory, + text, + fallback_position=phrase_index, + ) + score += max(0.0, placement_score) * 0.20 + phrase_candidates.append((score, preferred_position, phrase_index, text, tokens)) - for _, text, tokens in sorted(phrase_candidates, key=lambda item: item[0], reverse=True): + sorted_candidates = sorted(phrase_candidates, key=lambda item: item[0], reverse=True) + selected = [] + for score, preferred_position, phrase_index, text, tokens in sorted_candidates: key = text.lower() if key in phrase_seen: continue - prompt_phrases.append(text) + selected.append({ + "score": score, + "position": preferred_position, + "source_position": phrase_index, + "text": text, + "tokens": tokens, + }) phrase_seen.add(key) - if len(prompt_phrases) >= 12: + if len(selected) >= 12: break + prompt_phrases = [ + item["text"] + for item in sorted( + selected, + key=lambda item: (item["position"], -item["score"], item["source_position"], item["text"]), + ) + ] if not prompt_phrases: prompt_phrases = prompt_tokens @@ -1513,6 +1610,40 @@ class FunPackVideoRefiner: "phrases": prompt_phrases, } + def _lucky_phrase_preferred_position(self, placement_memory, phrase_text, fallback_position=0): + phrase_key = str(phrase_text).strip().lower() + entry = placement_memory.get(phrase_key) if isinstance(placement_memory, dict) else None + if not isinstance(entry, dict): + return int(fallback_position), 0.0 + positions = entry.get("positions") + if not isinstance(positions, dict) or not positions: + return int(fallback_position), 0.0 + + candidates = [] + for pos_key, item in positions.items(): + if not isinstance(item, dict): + continue + try: + pos = max(0, min(31, int(pos_key))) + except (TypeError, ValueError): + continue + score = self._phrase_position_score(item) + count_bonus = min(0.30, math.log1p(float(item.get("count", 0))) * 0.08) + candidates.append((score + count_bonus, pos)) + + if not candidates: + return int(fallback_position), 0.0 + + candidates = sorted(candidates, key=lambda item: item[0], reverse=True) + top_score = candidates[0][0] + close = [item for item in candidates if item[0] >= top_score - 0.18] + if len(close) > 1 and random.random() < 0.35: + weights = torch.tensor([max(0.05, item[0] + 1.25) for item in close], dtype=torch.float32) + selected = close[int(torch.multinomial(weights, 1).item())] + else: + selected = candidates[0] + return int(selected[1]), float(selected[0]) + def _encode_lucky_prompt_conditioning(self, clip, prompt_text, template, device, dtype): if clip is None or not prompt_text: return None, None, "" @@ -3942,6 +4073,7 @@ class FunPackVideoRefiner: "void_token_bank": {}, "void_token_pairs": {}, "lucky_context_memory": {}, + "lucky_phrase_placements": {}, "mode": mode, "scheduler_mode": scheduler_mode, "prodigy_d": {}, @@ -4036,6 +4168,7 @@ class FunPackVideoRefiner: self._ensure_void_token_bank(global_adaptive) self._ensure_void_pair_bank(global_adaptive) self._ensure_lucky_context_memory(global_adaptive) + self._ensure_lucky_phrase_placements(global_adaptive) global_adaptive.setdefault("mode", mode) self._ensure_sigma_state_defaults(global_adaptive) for cid in list(global_adaptive["concept_clusters"].keys()): @@ -4241,9 +4374,12 @@ class FunPackVideoRefiner: rated_positive = self._history_modified_conditioning(history[-1], source_reference, device) bank_rated_prompt = analysis_prompt + bank_rated_lucky_prompt = "" bank_rated_word_groups = word_groups bank_rated_positive = rated_positive bank_rated_token_mask = active_token_mask + if history and isinstance(history[-1].get("lucky"), dict): + bank_rated_lucky_prompt = str(history[-1].get("lucky", {}).get("prompt", "")).strip() if previous_prompt_key_for_rating in prompt_histories: previous_active = prompt_histories.get(previous_prompt_key_for_rating) @@ -4277,6 +4413,8 @@ class FunPackVideoRefiner: ) if previous_word_groups: bank_rated_prompt = previous_prompt + if isinstance(previous_history[-1].get("lucky"), dict): + bank_rated_lucky_prompt = str(previous_history[-1].get("lucky", {}).get("prompt", "")).strip() bank_rated_word_groups = previous_word_groups bank_rated_positive = previous_positive bank_rated_token_mask = previous_token_mask @@ -4290,6 +4428,9 @@ class FunPackVideoRefiner: iter_num, token_mask=bank_rated_token_mask, ) + self._update_lucky_phrase_placements(global_adaptive, bank_rated_prompt, rating_key, iter_num) + if bank_rated_lucky_prompt: + self._update_lucky_phrase_placements(global_adaptive, bank_rated_lucky_prompt, rating_key, iter_num) should_seed_current_prompt_after_lucky = ( bool(analysis_prompt) and @@ -4670,6 +4811,7 @@ class FunPackVideoRefiner: iter_num, token_mask=active_token_mask, ) + self._update_lucky_phrase_placements(global_adaptive, analysis_prompt, "discover", iter_num) # Update momentum momentum = 0.75 * momentum + 0.25 * (new_delta * reward) diff --git a/docs/FunPackGemmaEmbeddingRefiner.md b/docs/FunPackGemmaEmbeddingRefiner.md index 7324c1c7..fed3f2ad 100644 --- a/docs/FunPackGemmaEmbeddingRefiner.md +++ b/docs/FunPackGemmaEmbeddingRefiner.md @@ -96,7 +96,7 @@ Use it when you want to discover surprising concept-concept pairs or learn which Lucky stores token, pair, context, prompt-order, and conditioning-canvas memory without a fixed Lucky cap. Tokens stay available unless their embeddings are invalid; bad ratings teach the refiner to avoid specific adjacent token pairs or contexts first. With changing prompts, the previous generation's rating is applied to the previous prompt and its injected Lucky tokens even if Lucky expanded the output shape, then Lucky composes the current generation from that already-learned memory. The current prompt contributes fresh neutral tokens only after current Lucky composition, so new words become material for future runs rather than steering the same run. It can run by itself or alongside `into_the_void`. -For performance, Lucky keeps storage uncapped but does not decode the whole memory every run. Canvas selection uses saved tensor metadata first and only decodes the chosen canvas. Token composition chooses from the full eligible bank, then decodes only the token vectors actually selected for the current generation. When `clip` is connected, the learned prompt is capped to a practical per-run concept count before re-encoding so the text encoder is not asked to process the whole memory. The CLIP/Gemma prompt path prefers learned comma/semicolon-separated concept phrases, falling back to loose tokens only when phrase memory is unavailable. Lucky runs also share one stable memory history inside the refinement session, so changing enhancer text does not force normal prompt-variant conditioning scans or create a new heavy prompt history on every run. +For performance, Lucky keeps storage uncapped but does not decode the whole memory every run. Canvas selection uses saved tensor metadata first and only decodes the chosen canvas. Token composition chooses from the full eligible bank, then decodes only the token vectors actually selected for the current generation. When `clip` is connected, the learned prompt is capped to a practical per-run concept count before re-encoding so the text encoder is not asked to process the whole memory. The CLIP/Gemma prompt path prefers learned comma/semicolon-separated concept phrases, falling back to loose tokens only when phrase memory is unavailable. It also tracks the rated position where each phrase tends to work best, then sorts combined Lucky phrases back toward those learned positions. If a phrase works in several positions, Lucky can choose between close placements with a rating-weighted random pick. Lucky runs also share one stable memory history inside the refinement session, so changing enhancer text does not force normal prompt-variant conditioning scans or create a new heavy prompt history on every run. ## Latent Refinement