From 7315fab45915bffb86a0362e68e75da5c73bb88e Mon Sep 17 00:00:00 2001 From: aimfordeb <133010426+aimfordeb@users.noreply.github.com> Date: Thu, 23 Apr 2026 14:52:03 +0300 Subject: [PATCH] Simplify FunPack LoRA weight workflow --- CHANGELOG.md | 2 +- README.md | 6 +- __init__.py | 3 + conditioning.py | 22 ++--- docs/FunPackLoraWorkflow.md | 8 +- model_management.py | 163 ++++++++++++++++++++++++------------ web/funpack_lora_weights.js | 134 +++++++++++++++++++++++++++++ 7 files changed, 261 insertions(+), 77 deletions(-) create mode 100644 web/funpack_lora_weights.js diff --git a/CHANGELOG.md b/CHANGELOG.md index eaf79d9f..22ef7177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ Split the old single `funpack.py` implementation into focused modules: `funpack.py` remains as a compatibility re-export for older imports. -Updated `FunPack Gemma Embedding Refiner` so it can accept a FunPack LoRA stack and save next-run LoRA weight suggestions based on prompt concepts, LoRA concept/type hints, and user ratings. +Updated `FunPack Gemma Embedding Refiner` so it can accept a FunPack LoRA stack and save next-run model LoRA weight suggestions based on prompt concepts, LoRA type hints, and user ratings. ## [1.3.3] - 2026-04-22 diff --git a/README.md b/README.md index 7b66d92a..e819e02f 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ Use this order: `FunPack Apply LoRA Weights` -> `FunPack LoRA Loader` -> `FunPack Gemma Embedding Refiner` -`FunPack Apply LoRA Weights` defines selected LoRAs, base weights, LoRA type, and LoRA concept hints. If the refiner has saved suggested weights for the exact prompt, it applies them; otherwise it uses the base weights. +`FunPack Apply LoRA Weights` defines selected LoRAs, LoRA type, and model base weights. If the refiner has saved suggested weights for the exact prompt, it applies them; otherwise it uses the base weights. -`FunPack LoRA Loader` only loads the prepared LoRA stack into the model and CLIP, then passes that stack forward. +`FunPack LoRA Loader` loads the prepared LoRA stack into the model. Its CLIP input is optional and is left untouched when omitted. `FunPack Gemma Embedding Refiner` does the prompt/concept/rating work and saves next-run LoRA weight suggestions into its existing refinement JSON. @@ -84,3 +84,5 @@ Version history is available in [CHANGELOG.md](CHANGELOG.md). ## Feedback If you have suggestions, questions, or ideas for new nodes, feel free to open an Issue or submit a pull request. + +Thanks to teams of OpenAI, xAI, DeepSeek, Anthropic and Google and their respective AI large language models for providing all the help with the code and transforming my stupid ideas into something actually running in your UI. Thanks to all testers and users who use FunPack in their workflows daily and provide feedback in any way. Without all of you, this would have been a dream of a wannabe coder nerd. diff --git a/__init__.py b/__init__.py index 89b27ee0..cdb33671 100644 --- a/__init__.py +++ b/__init__.py @@ -16,6 +16,8 @@ from .image_processing import ( from .model_management import FunPackApplyLoraWeights, FunPackLoraLoader from .samplers import FunPackHybridEuler2SSampler +WEB_DIRECTORY = "./web" + NODE_CLASS_MAPPINGS = { "FunPackPromptCombiner": FunPackPromptCombiner, "FunPackStoryMemKeyframeExtractor": FunPackStoryMemKeyframeExtractor, @@ -49,4 +51,5 @@ NODE_DISPLAY_NAME_MAPPINGS = { __all__ = [ "NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", + "WEB_DIRECTORY", ] diff --git a/conditioning.py b/conditioning.py index aa4f2500..eba2e1d0 100644 --- a/conditioning.py +++ b/conditioning.py @@ -1232,8 +1232,8 @@ class FunPackGemmaEmbeddingRefiner: # LORA WEIGHT SUGGESTIONS # ========================================================================= - def _lora_state_id(self, lora_name: str, lora_type: str, concept: str): - return md5(f"{lora_name}::{lora_type}::{concept}".encode("utf-8")).hexdigest()[:16] + def _lora_state_id(self, lora_name: str, lora_type: str): + return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16] def _lora_words(self, text: str): return { @@ -1244,14 +1244,12 @@ class FunPackGemmaEmbeddingRefiner: def _ensure_lora_memory(self, memory: dict, lora_entry: dict): lora_type = lora_entry.get("type", "general") - concept = lora_entry.get("concept", "") - lora_id = lora_entry.get("id") or self._lora_state_id(lora_entry.get("name", ""), lora_type, concept) + lora_id = lora_entry.get("id") or self._lora_state_id(lora_entry.get("name", ""), lora_type) state = memory.setdefault( lora_id, { "name": lora_entry.get("name", ""), "type": lora_type, - "concept": concept, "offset_ratio": 0.0, "stable_offset_ratio": None, "reward_ema": 0.0, @@ -1262,7 +1260,6 @@ class FunPackGemmaEmbeddingRefiner: ) state["name"] = lora_entry.get("name", "") state["type"] = lora_type - state["concept"] = concept state.setdefault("offset_ratio", 0.0) state.setdefault("stable_offset_ratio", None) state.setdefault("reward_ema", 0.0) @@ -1273,9 +1270,7 @@ class FunPackGemmaEmbeddingRefiner: def _score_lora_prompt_relation(self, lora_entry, ordered_concept_ids, concept_clusters, current_concept_labels): lora_type = lora_entry.get("type", "general") - concept_words = self._lora_words(lora_entry.get("concept", "")) - if not concept_words: - concept_words = self._lora_words(lora_entry.get("name", "")) + lora_words = self._lora_words(lora_entry.get("name", "")) best_score = 0.35 if lora_type == "general" else 0.0 best_labels = [] @@ -1292,7 +1287,7 @@ class FunPackGemmaEmbeddingRefiner: prompt_words = label_words | anchor_words category = cluster.get("category", "general") - overlap = len(concept_words & prompt_words) / max(1, len(concept_words)) if concept_words else 0.0 + overlap = len(lora_words & prompt_words) / max(1, len(lora_words)) if lora_words else 0.0 category_score = 0.62 if lora_type == category else 0.0 if lora_type == "quality" and category == "quality": category_score = 0.78 @@ -1376,17 +1371,12 @@ class FunPackGemmaEmbeddingRefiner: state["iterations"] = int(state.get("iterations", 0)) + 1 base_model = float(entry.get("base_model_weight", entry.get("model_weight", 1.0))) - base_clip = float(entry.get("base_clip_weight", entry.get("clip_weight", base_model))) model_weight = base_model * (1.0 + offset) - clip_weight = base_clip * (1.0 + offset) suggestions[lora_id] = { "name": entry.get("name", ""), "type": lora_type, - "concept": entry.get("concept", ""), "model_weight": model_weight, - "clip_weight": clip_weight, "base_model_weight": base_model, - "base_clip_weight": base_clip, "offset_ratio": offset, "relation": relation, "matched_concepts": matched_labels, @@ -1399,7 +1389,7 @@ class FunPackGemmaEmbeddingRefiner: match_text = ",".join(matched_labels) if matched_labels else "none" status_parts.append( f"{entry.get('name', '?')}[{lora_type}] rel={relation:.2f} " - f"offset={offset:+.3f} next=({model_weight:+.3f},{clip_weight:+.3f}) " + f"offset={offset:+.3f} next={model_weight:+.3f} " f"match={match_text}" ) diff --git a/docs/FunPackLoraWorkflow.md b/docs/FunPackLoraWorkflow.md index b17ee516..bafb9d53 100644 --- a/docs/FunPackLoraWorkflow.md +++ b/docs/FunPackLoraWorkflow.md @@ -21,21 +21,21 @@ Inputs: - **mode**: Same tokenizer mode as the refiner. - **lora_N**: LoRA file. - **lora_N_type**: `general`, `concept`, `style`, `quality`, or `character`. -- **lora_N_concept**: Human-readable concept hint for the refiner. - **lora_N_base_weight**: Trainer-recommended model base weight. -- **lora_N_clip_base_weight**: Trainer-recommended CLIP base weight. + +Use **+ Add LoRA** to add more LoRA rows as needed. On the first run for an exact prompt, no saved suggestion exists, so base weights are used. On later runs, the node reads the refiner JSON and applies saved suggestions for that exact prompt only. ## FunPack LoRA Loader -This node only loads the LoRA stack. It applies the model and CLIP weights prepared by `FunPack Apply LoRA Weights`, then passes the same stack forward so the refiner can learn from it. +This node only loads the LoRA stack. It applies the model weights prepared by `FunPack Apply LoRA Weights`, then passes the same stack forward so the refiner can learn from it. The `clip` input is optional and uses zero CLIP strength when omitted. ## Gemma Refiner Integration Connect the `lora_stack` output from `FunPack LoRA Loader` into the optional `lora_stack` input on `FunPack Gemma Embedding Refiner`. -The refiner compares each LoRA's declared type/concept with the concepts it already extracted from the prompt. After processing the rating, it saves `lora_weight_suggestions` into the same prompt entry in the refiner JSON. +The refiner compares each LoRA's declared type and filename with the concepts it already extracted from the prompt. After processing the rating, it saves `lora_weight_suggestions` into the same prompt entry in the refiner JSON. Examples: diff --git a/model_management.py b/model_management.py index 1142c24d..b7efefbf 100644 --- a/model_management.py +++ b/model_management.py @@ -9,10 +9,47 @@ import folder_paths LORA_TYPES = ["general", "concept", "style", "quality", "character"] -LORA_SLOT_COUNT = 8 LORA_STACK_TYPE = "FUNPACK_LORA_STACK" +class AnyType(str): + def __ne__(self, _): + return False + + +any_type = AnyType("*") + + +class FlexibleOptionalInputType(dict): + def __init__(self, input_type, data=None): + self.input_type = input_type + self.data = data or {} + + def __contains__(self, _): + return True + + def __getitem__(self, key): + return self.data.get(key, (self.input_type,)) + + def get(self, key, default=None): + return self.data.get(key, default) + + def items(self): + return self.data.items() + + def keys(self): + return self.data.keys() + + def values(self): + return self.data.values() + + def __iter__(self): + return iter(self.data) + + def __len__(self): + return len(self.data) + + def normalize_prompt_for_mode(prompt, mode): prompt = (prompt or "").strip() if (mode or "ltx2").lower() == "wan": @@ -26,8 +63,8 @@ def prompt_key_for_mode(prompt, mode): return prompt or "" -def lora_state_id(lora_name, lora_type, concept): - return md5(f"{lora_name}::{lora_type}::{concept}".encode("utf-8")).hexdigest()[:16] +def lora_state_id(lora_name, lora_type): + return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16] def refiner_state_path(refinement_key, mode): @@ -52,38 +89,23 @@ class FunPackApplyLoraWeights: @classmethod def INPUT_TYPES(cls): loras = ["None"] + folder_paths.get_filename_list("loras") - optional = {} - for index in range(LORA_SLOT_COUNT): - optional[f"lora_{index}"] = (loras, {"default": "None"}) - optional[f"lora_{index}_type"] = (LORA_TYPES, {"default": "general"}) - optional[f"lora_{index}_concept"] = ( - "STRING", - { - "default": "", - "multiline": False, - "tooltip": "Concept this LoRA is meant to affect. Gemma Refiner uses this hint.", - }, - ) - optional[f"lora_{index}_base_weight"] = ( - "FLOAT", - { - "default": 1.0, - "min": -10.0, - "max": 10.0, - "step": 0.01, - "tooltip": "Trainer-recommended model anchor weight.", - }, - ) - optional[f"lora_{index}_clip_base_weight"] = ( - "FLOAT", - { - "default": 1.0, - "min": -10.0, - "max": 10.0, - "step": 0.01, - "tooltip": "Trainer-recommended CLIP anchor weight.", - }, - ) + optional = FlexibleOptionalInputType( + any_type, + { + "lora_0": (loras, {"default": "None"}), + "lora_0_type": (LORA_TYPES, {"default": "general"}), + "lora_0_base_weight": ( + "FLOAT", + { + "default": 1.0, + "min": -10.0, + "max": 10.0, + "step": 0.01, + "tooltip": "Trainer-recommended model anchor weight.", + }, + ), + }, + ) return { "required": { @@ -120,26 +142,62 @@ class FunPackApplyLoraWeights: return suggestions, "refiner suggestions applied" def _iter_slots(self, kwargs): - for index in range(LORA_SLOT_COUNT): + indexed_slots = set() + for key in kwargs: + match = re.fullmatch(r"lora_(\d+)", key) + if match: + indexed_slots.add(int(match.group(1))) + + for index in sorted(indexed_slots): lora_name = kwargs.get(f"lora_{index}", "None") + if isinstance(lora_name, dict): + if not lora_name.get("on", True): + continue + name = lora_name.get("lora", "None") + if not name or name == "None": + continue + lora_type = lora_name.get("type", lora_name.get("lora_type", "general")) + if lora_type not in LORA_TYPES: + lora_type = "general" + yield { + "slot": index, + "name": name, + "type": lora_type, + "id": lora_state_id(name, lora_type), + "base_model_weight": float(lora_name.get("strength", lora_name.get("base_weight", 1.0))), + } + continue + if not lora_name or lora_name == "None": continue lora_type = kwargs.get(f"lora_{index}_type", "general") if lora_type not in LORA_TYPES: lora_type = "general" - concept = (kwargs.get(f"lora_{index}_concept", "") or "").strip() yield { "slot": index, "name": lora_name, "type": lora_type, - "concept": concept, - "id": lora_state_id(lora_name, lora_type, concept), + "id": lora_state_id(lora_name, lora_type), "base_model_weight": float(kwargs.get(f"lora_{index}_base_weight", 1.0)), - "base_clip_weight": float(kwargs.get(f"lora_{index}_clip_base_weight", 1.0)), } + def _get_suggestion(self, suggestions, entry): + suggestion = suggestions.get(entry["id"]) + if suggestion: + return suggestion + + for legacy in suggestions.values(): + if ( + isinstance(legacy, dict) + and legacy.get("name") == entry["name"] + and legacy.get("type", "general") == entry["type"] + ): + return legacy + + return {} + def apply_lora_weights(self, positive_prompt, refinement_key, mode, **kwargs): mode = (mode or "ltx2").lower() prompt_key = prompt_key_for_mode(positive_prompt, mode) @@ -148,22 +206,18 @@ class FunPackApplyLoraWeights: loras = [] lines = [f"FunPack Apply LoRA Weights | {source_message}"] for entry in self._iter_slots(kwargs): - suggestion = suggestions.get(entry["id"], {}) + suggestion = self._get_suggestion(suggestions, entry) model_weight = float(suggestion.get("model_weight", entry["base_model_weight"])) - clip_weight = float(suggestion.get("clip_weight", entry["base_clip_weight"])) source = "suggested" if suggestion else "base" stack_entry = dict(entry) stack_entry["model_weight"] = model_weight - stack_entry["clip_weight"] = clip_weight stack_entry["source"] = source loras.append(stack_entry) - concept = entry["concept"] or entry["type"] lines.append( - f"lora_{entry['slot']}: {entry['name']} [{entry['type']}:{concept}] " - f"{source}=({model_weight:+.3f}, {clip_weight:+.3f}) " - f"base=({entry['base_model_weight']:+.3f}, {entry['base_clip_weight']:+.3f})" + f"lora_{entry['slot']}: {entry['name']} [{entry['type']}] " + f"{source}={model_weight:+.3f} base={entry['base_model_weight']:+.3f}" ) stack = { @@ -198,9 +252,11 @@ class FunPackLoraLoader: return { "required": { "model": ("MODEL",), - "clip": ("CLIP",), "lora_stack": (LORA_STACK_TYPE,), - } + }, + "optional": { + "clip": ("CLIP",), + }, } def _load_lora_file(self, lora_name): @@ -218,24 +274,23 @@ class FunPackLoraLoader: return lora - def load_loras(self, model, clip, lora_stack): + def load_loras(self, model, lora_stack, clip=None): loras = lora_stack.get("loras", []) if isinstance(lora_stack, dict) else [] lines = [f"FunPack LoRA Loader | loading {len(loras)} LoRA(s)"] loaded_count = 0 for entry in loras: model_weight = float(entry.get("model_weight", 0.0)) - clip_weight = float(entry.get("clip_weight", 0.0)) - if model_weight == 0 and clip_weight == 0: + if model_weight == 0: lines.append(f"lora_{entry.get('slot', '?')}: {entry.get('name', '?')} skipped at zero weight") continue lora = self._load_lora_file(entry["name"]) - model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_weight, clip_weight) + model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_weight, 0.0) loaded_count += 1 lines.append( f"lora_{entry.get('slot', '?')}: {entry['name']} " - f"applied=({model_weight:+.3f}, {clip_weight:+.3f}) source={entry.get('source', 'base')}" + f"applied={model_weight:+.3f} source={entry.get('source', 'base')}" ) if loaded_count == 0: diff --git a/web/funpack_lora_weights.js b/web/funpack_lora_weights.js new file mode 100644 index 00000000..61c2e6ab --- /dev/null +++ b/web/funpack_lora_weights.js @@ -0,0 +1,134 @@ +import { app } from "../../scripts/app.js"; + +const NODE_NAME = "FunPackApplyLoraWeights"; +const LORA_TYPES = ["general", "concept", "style", "quality", "character"]; +const ADD_BUTTON_NAME = "+ Add LoRA"; + +function getLoraValues(nodeData) { + return nodeData?.input?.optional?.lora_0?.[0] || ["None"]; +} + +function getNextLoraIndex(node) { + let next = 0; + for (const widget of node.widgets || []) { + const match = /^lora_(\d+)$/.exec(widget.name || ""); + if (match) { + next = Math.max(next, Number(match[1]) + 1); + } + } + return next; +} + +function moveAddButtonToEnd(node) { + const button = (node.widgets || []).find((widget) => widget.name === ADD_BUTTON_NAME); + if (!button) { + return; + } + const index = node.widgets.indexOf(button); + if (index >= 0 && index !== node.widgets.length - 1) { + node.widgets.splice(index, 1); + node.widgets.push(button); + } +} + +function resizeNode(node) { + const computed = node.computeSize(); + node.size[0] = Math.max(node.size[0], computed[0]); + node.size[1] = computed[1]; +} + +function addLoraRow(node, nodeData, value = {}, markDirty = true) { + const index = getNextLoraIndex(node); + const loraWidget = node.addWidget( + "combo", + `lora_${index}`, + value.lora || "None", + undefined, + { values: getLoraValues(nodeData) }, + ); + const typeWidget = node.addWidget( + "combo", + `lora_${index}_type`, + value.type || "general", + undefined, + { values: LORA_TYPES }, + ); + const weightWidget = node.addWidget( + "number", + `lora_${index}_base_weight`, + value.base_weight ?? 1.0, + undefined, + { min: -10.0, max: 10.0, step: 0.01, precision: 3 }, + ); + + loraWidget.serialize = true; + typeWidget.serialize = true; + weightWidget.serialize = true; + moveAddButtonToEnd(node); + resizeNode(node); + if (markDirty) { + node.setDirtyCanvas(true, true); + } +} + +function ensureAddButton(node, nodeData) { + if ((node.widgets || []).some((widget) => widget.name === ADD_BUTTON_NAME)) { + moveAddButtonToEnd(node); + return; + } + + const button = node.addWidget("button", ADD_BUTTON_NAME, null, () => { + addLoraRow(node, nodeData); + }); + button.serialize = false; + moveAddButtonToEnd(node); + resizeNode(node); +} + +function restoreExtraRows(node, nodeData, info) { + const values = info?.widgets_values; + if (!Array.isArray(values)) { + return; + } + + node.widgets = (node.widgets || []).filter((widget) => { + const match = /^lora_(\d+)(?:_type|_base_weight)?$/.exec(widget.name || ""); + return !match || Number(match[1]) === 0; + }); + + const knownWidgetCount = 6; + for (let offset = knownWidgetCount; offset + 2 < values.length; offset += 3) { + addLoraRow( + node, + nodeData, + { + lora: values[offset], + type: values[offset + 1], + base_weight: values[offset + 2], + }, + false, + ); + } +} + +app.registerExtension({ + name: "funpack.loraWeights", + beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== NODE_NAME) { + return; + } + + const originalOnNodeCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + originalOnNodeCreated?.apply(this, arguments); + ensureAddButton(this, nodeData); + }; + + const originalConfigure = nodeType.prototype.configure; + nodeType.prototype.configure = function (info) { + originalConfigure?.apply(this, arguments); + ensureAddButton(this, nodeData); + restoreExtraRows(this, nodeData, info); + }; + }, +});