nodes/olivv-cs-ComfyUI-FunPack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simplify FunPack LoRA weight workflow

Browse Source
This commit is contained in:
aimfordeb
2026-04-23 14:52:03 +03:00
parent 0f0d726b1f
commit 7315fab459
7 changed files with 261 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1 -1
View File
@@ -19,7 +19,7 @@ Split the old single `funpack.py` implementation into focused modules:
`funpack.py` remains as a compatibility re-export for older imports.
Updated `FunPack Gemma Embedding Refiner` so it can accept a FunPack LoRA stack and save next-run LoRA weight suggestions based on prompt concepts, LoRA concept/type hints, and user ratings.
Updated `FunPack Gemma Embedding Refiner` so it can accept a FunPack LoRA stack and save next-run model LoRA weight suggestions based on prompt concepts, LoRA type hints, and user ratings.
## [1.3.3] - 2026-04-22
README.md
+4 -2
View File
@@ -17,9 +17,9 @@ Use this order:
`FunPack Apply LoRA Weights` -> `FunPack LoRA Loader` -> `FunPack Gemma Embedding Refiner`
`FunPack Apply LoRA Weights` defines selected LoRAs, base weights, LoRA type, and LoRA concept hints. If the refiner has saved suggested weights for the exact prompt, it applies them; otherwise it uses the base weights.
`FunPack Apply LoRA Weights` defines selected LoRAs, LoRA type, and model base weights. If the refiner has saved suggested weights for the exact prompt, it applies them; otherwise it uses the base weights.
`FunPack LoRA Loader` only loads the prepared LoRA stack into the model and CLIP, then passes that stack forward.
`FunPack LoRA Loader` loads the prepared LoRA stack into the model. Its CLIP input is optional and is left untouched when omitted.
`FunPack Gemma Embedding Refiner` does the prompt/concept/rating work and saves next-run LoRA weight suggestions into its existing refinement JSON.
@@ -84,3 +84,5 @@ Version history is available in [CHANGELOG.md](CHANGELOG.md).
## Feedback
If you have suggestions, questions, or ideas for new nodes, feel free to open an Issue or submit a pull request.
Thanks to teams of OpenAI, xAI, DeepSeek, Anthropic and Google and their respective AI large language models for providing all the help with the code and transforming my stupid ideas into something actually running in your UI. Thanks to all testers and users who use FunPack in their workflows daily and provide feedback in any way. Without all of you, this would have been a dream of a wannabe coder nerd.
__init__.py
+3
View File
@@ -16,6 +16,8 @@ from .image_processing import (
from .model_management import FunPackApplyLoraWeights, FunPackLoraLoader
from .samplers import FunPackHybridEuler2SSampler
WEB_DIRECTORY = "./web"
NODE_CLASS_MAPPINGS = {
"FunPackPromptCombiner": FunPackPromptCombiner,
"FunPackStoryMemKeyframeExtractor": FunPackStoryMemKeyframeExtractor,
@@ -49,4 +51,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
__all__ = [
"NODE_CLASS_MAPPINGS",
"NODE_DISPLAY_NAME_MAPPINGS",
"WEB_DIRECTORY",
]
conditioning.py
+6 -16
View File
@@ -1232,8 +1232,8 @@ class FunPackGemmaEmbeddingRefiner:
# LORA WEIGHT SUGGESTIONS
# =========================================================================
def _lora_state_id(self, lora_name: str, lora_type: str, concept: str):
return md5(f"{lora_name}::{lora_type}::{concept}".encode("utf-8")).hexdigest()[:16]
def _lora_state_id(self, lora_name: str, lora_type: str):
return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16]
def _lora_words(self, text: str):
return {
@@ -1244,14 +1244,12 @@ class FunPackGemmaEmbeddingRefiner:
def _ensure_lora_memory(self, memory: dict, lora_entry: dict):
lora_type = lora_entry.get("type", "general")
concept = lora_entry.get("concept", "")
lora_id = lora_entry.get("id") or self._lora_state_id(lora_entry.get("name", ""), lora_type, concept)
lora_id = lora_entry.get("id") or self._lora_state_id(lora_entry.get("name", ""), lora_type)
state = memory.setdefault(
lora_id,
{
"name": lora_entry.get("name", ""),
"type": lora_type,
"concept": concept,
"offset_ratio": 0.0,
"stable_offset_ratio": None,
"reward_ema": 0.0,
@@ -1262,7 +1260,6 @@ class FunPackGemmaEmbeddingRefiner:
)
state["name"] = lora_entry.get("name", "")
state["type"] = lora_type
state["concept"] = concept
state.setdefault("offset_ratio", 0.0)
state.setdefault("stable_offset_ratio", None)
state.setdefault("reward_ema", 0.0)
@@ -1273,9 +1270,7 @@ class FunPackGemmaEmbeddingRefiner:
def _score_lora_prompt_relation(self, lora_entry, ordered_concept_ids, concept_clusters, current_concept_labels):
lora_type = lora_entry.get("type", "general")
concept_words = self._lora_words(lora_entry.get("concept", ""))
if not concept_words:
concept_words = self._lora_words(lora_entry.get("name", ""))
lora_words = self._lora_words(lora_entry.get("name", ""))
best_score = 0.35 if lora_type == "general" else 0.0
best_labels = []
@@ -1292,7 +1287,7 @@ class FunPackGemmaEmbeddingRefiner:
prompt_words = label_words | anchor_words
category = cluster.get("category", "general")
overlap = len(concept_words & prompt_words) / max(1, len(concept_words)) if concept_words else 0.0
overlap = len(lora_words & prompt_words) / max(1, len(lora_words)) if lora_words else 0.0
category_score = 0.62 if lora_type == category else 0.0
if lora_type == "quality" and category == "quality":
category_score = 0.78
@@ -1376,17 +1371,12 @@ class FunPackGemmaEmbeddingRefiner:
state["iterations"] = int(state.get("iterations", 0)) + 1
base_model = float(entry.get("base_model_weight", entry.get("model_weight", 1.0)))
base_clip = float(entry.get("base_clip_weight", entry.get("clip_weight", base_model)))
model_weight = base_model * (1.0 + offset)
clip_weight = base_clip * (1.0 + offset)
suggestions[lora_id] = {
"name": entry.get("name", ""),
"type": lora_type,
"concept": entry.get("concept", ""),
"model_weight": model_weight,
"clip_weight": clip_weight,
"base_model_weight": base_model,
"base_clip_weight": base_clip,
"offset_ratio": offset,
"relation": relation,
"matched_concepts": matched_labels,
@@ -1399,7 +1389,7 @@ class FunPackGemmaEmbeddingRefiner:
match_text = ",".join(matched_labels) if matched_labels else "none"
status_parts.append(
f"{entry.get('name', '?')}[{lora_type}] rel={relation:.2f} "
f"offset={offset:+.3f} next=({model_weight:+.3f},{clip_weight:+.3f}) "
f"offset={offset:+.3f} next={model_weight:+.3f} "
f"match={match_text}"
)
docs/FunPackLoraWorkflow.md
+4 -4
View File
@@ -21,21 +21,21 @@ Inputs:
- **mode**: Same tokenizer mode as the refiner.
- **lora_N**: LoRA file.
- **lora_N_type**: `general`, `concept`, `style`, `quality`, or `character`.
- **lora_N_concept**: Human-readable concept hint for the refiner.
- **lora_N_base_weight**: Trainer-recommended model base weight.
- **lora_N_clip_base_weight**: Trainer-recommended CLIP base weight.
Use **+ Add LoRA** to add more LoRA rows as needed.
On the first run for an exact prompt, no saved suggestion exists, so base weights are used. On later runs, the node reads the refiner JSON and applies saved suggestions for that exact prompt only.
## FunPack LoRA Loader
This node only loads the LoRA stack. It applies the model and CLIP weights prepared by `FunPack Apply LoRA Weights`, then passes the same stack forward so the refiner can learn from it.
This node only loads the LoRA stack. It applies the model weights prepared by `FunPack Apply LoRA Weights`, then passes the same stack forward so the refiner can learn from it. The `clip` input is optional and uses zero CLIP strength when omitted.
## Gemma Refiner Integration
Connect the `lora_stack` output from `FunPack LoRA Loader` into the optional `lora_stack` input on `FunPack Gemma Embedding Refiner`.
The refiner compares each LoRA's declared type/concept with the concepts it already extracted from the prompt. After processing the rating, it saves `lora_weight_suggestions` into the same prompt entry in the refiner JSON.
The refiner compares each LoRA's declared type and filename with the concepts it already extracted from the prompt. After processing the rating, it saves `lora_weight_suggestions` into the same prompt entry in the refiner JSON.
Examples:
model_management.py
+109 -54
View File
@@ -9,10 +9,47 @@ import folder_paths
LORA_TYPES = ["general", "concept", "style", "quality", "character"]
LORA_SLOT_COUNT = 8
LORA_STACK_TYPE = "FUNPACK_LORA_STACK"
class AnyType(str):
def __ne__(self, _):
return False
any_type = AnyType("*")
class FlexibleOptionalInputType(dict):
def __init__(self, input_type, data=None):
self.input_type = input_type
self.data = data or {}
def __contains__(self, _):
return True
def __getitem__(self, key):
return self.data.get(key, (self.input_type,))
def get(self, key, default=None):
return self.data.get(key, default)
def items(self):
return self.data.items()
def keys(self):
return self.data.keys()
def values(self):
return self.data.values()
def __iter__(self):
return iter(self.data)
def __len__(self):
return len(self.data)
def normalize_prompt_for_mode(prompt, mode):
prompt = (prompt or "").strip()
if (mode or "ltx2").lower() == "wan":
@@ -26,8 +63,8 @@ def prompt_key_for_mode(prompt, mode):
return prompt or ""
def lora_state_id(lora_name, lora_type, concept):
return md5(f"{lora_name}::{lora_type}::{concept}".encode("utf-8")).hexdigest()[:16]
def lora_state_id(lora_name, lora_type):
return md5(f"{lora_name}::{lora_type}".encode("utf-8")).hexdigest()[:16]
def refiner_state_path(refinement_key, mode):
@@ -52,38 +89,23 @@ class FunPackApplyLoraWeights:
@classmethod
def INPUT_TYPES(cls):
loras = ["None"] + folder_paths.get_filename_list("loras")
optional = {}
for index in range(LORA_SLOT_COUNT):
optional[f"lora_{index}"] = (loras, {"default": "None"})
optional[f"lora_{index}_type"] = (LORA_TYPES, {"default": "general"})
optional[f"lora_{index}_concept"] = (
"STRING",
{
"default": "",
"multiline": False,
"tooltip": "Concept this LoRA is meant to affect. Gemma Refiner uses this hint.",
},
)
optional[f"lora_{index}_base_weight"] = (
"FLOAT",
{
"default": 1.0,
"min": -10.0,
"max": 10.0,
"step": 0.01,
"tooltip": "Trainer-recommended model anchor weight.",
},
)
optional[f"lora_{index}_clip_base_weight"] = (
"FLOAT",
{
"default": 1.0,
"min": -10.0,
"max": 10.0,
"step": 0.01,
"tooltip": "Trainer-recommended CLIP anchor weight.",
},
)
optional = FlexibleOptionalInputType(
any_type,
{
"lora_0": (loras, {"default": "None"}),
"lora_0_type": (LORA_TYPES, {"default": "general"}),
"lora_0_base_weight": (
"FLOAT",
{
"default": 1.0,
"min": -10.0,
"max": 10.0,
"step": 0.01,
"tooltip": "Trainer-recommended model anchor weight.",
},
),
},
)
return {
"required": {
@@ -120,26 +142,62 @@ class FunPackApplyLoraWeights:
return suggestions, "refiner suggestions applied"
def _iter_slots(self, kwargs):
for index in range(LORA_SLOT_COUNT):
indexed_slots = set()
for key in kwargs:
match = re.fullmatch(r"lora_(\d+)", key)
if match:
indexed_slots.add(int(match.group(1)))
for index in sorted(indexed_slots):
lora_name = kwargs.get(f"lora_{index}", "None")
if isinstance(lora_name, dict):
if not lora_name.get("on", True):
continue
name = lora_name.get("lora", "None")
if not name or name == "None":
continue
lora_type = lora_name.get("type", lora_name.get("lora_type", "general"))
if lora_type not in LORA_TYPES:
lora_type = "general"
yield {
"slot": index,
"name": name,
"type": lora_type,
"id": lora_state_id(name, lora_type),
"base_model_weight": float(lora_name.get("strength", lora_name.get("base_weight", 1.0))),
}
continue
if not lora_name or lora_name == "None":
continue
lora_type = kwargs.get(f"lora_{index}_type", "general")
if lora_type not in LORA_TYPES:
lora_type = "general"
concept = (kwargs.get(f"lora_{index}_concept", "") or "").strip()
yield {
"slot": index,
"name": lora_name,
"type": lora_type,
"concept": concept,
"id": lora_state_id(lora_name, lora_type, concept),
"id": lora_state_id(lora_name, lora_type),
"base_model_weight": float(kwargs.get(f"lora_{index}_base_weight", 1.0)),
"base_clip_weight": float(kwargs.get(f"lora_{index}_clip_base_weight", 1.0)),
}
def _get_suggestion(self, suggestions, entry):
suggestion = suggestions.get(entry["id"])
if suggestion:
return suggestion
for legacy in suggestions.values():
if (
isinstance(legacy, dict)
and legacy.get("name") == entry["name"]
and legacy.get("type", "general") == entry["type"]
):
return legacy
return {}
def apply_lora_weights(self, positive_prompt, refinement_key, mode, **kwargs):
mode = (mode or "ltx2").lower()
prompt_key = prompt_key_for_mode(positive_prompt, mode)
@@ -148,22 +206,18 @@ class FunPackApplyLoraWeights:
loras = []
lines = [f"FunPack Apply LoRA Weights | {source_message}"]
for entry in self._iter_slots(kwargs):
suggestion = suggestions.get(entry["id"], {})
suggestion = self._get_suggestion(suggestions, entry)
model_weight = float(suggestion.get("model_weight", entry["base_model_weight"]))
clip_weight = float(suggestion.get("clip_weight", entry["base_clip_weight"]))
source = "suggested" if suggestion else "base"
stack_entry = dict(entry)
stack_entry["model_weight"] = model_weight
stack_entry["clip_weight"] = clip_weight
stack_entry["source"] = source
loras.append(stack_entry)
concept = entry["concept"] or entry["type"]
lines.append(
f"lora_{entry['slot']}: {entry['name']} [{entry['type']}:{concept}] "
f"{source}=({model_weight:+.3f}, {clip_weight:+.3f}) "
f"base=({entry['base_model_weight']:+.3f}, {entry['base_clip_weight']:+.3f})"
f"lora_{entry['slot']}: {entry['name']} [{entry['type']}] "
f"{source}={model_weight:+.3f} base={entry['base_model_weight']:+.3f}"
)
stack = {
@@ -198,9 +252,11 @@ class FunPackLoraLoader:
return {
"required": {
"model": ("MODEL",),
"clip": ("CLIP",),
"lora_stack": (LORA_STACK_TYPE,),
}
},
"optional": {
"clip": ("CLIP",),
},
}
def _load_lora_file(self, lora_name):
@@ -218,24 +274,23 @@ class FunPackLoraLoader:
return lora
def load_loras(self, model, clip, lora_stack):
def load_loras(self, model, lora_stack, clip=None):
loras = lora_stack.get("loras", []) if isinstance(lora_stack, dict) else []
lines = [f"FunPack LoRA Loader | loading {len(loras)} LoRA(s)"]
loaded_count = 0
for entry in loras:
model_weight = float(entry.get("model_weight", 0.0))
clip_weight = float(entry.get("clip_weight", 0.0))
if model_weight == 0 and clip_weight == 0:
if model_weight == 0:
lines.append(f"lora_{entry.get('slot', '?')}: {entry.get('name', '?')} skipped at zero weight")
continue
lora = self._load_lora_file(entry["name"])
model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_weight, clip_weight)
model, clip = comfy.sd.load_lora_for_models(model, clip, lora, model_weight, 0.0)
loaded_count += 1
lines.append(
f"lora_{entry.get('slot', '?')}: {entry['name']} "
f"applied=({model_weight:+.3f}, {clip_weight:+.3f}) source={entry.get('source', 'base')}"
f"applied={model_weight:+.3f} source={entry.get('source', 'base')}"
)
if loaded_count == 0:
web/funpack_lora_weights.js
+134
View File
@@ -0,0 +1,134 @@
import { app } from "../../scripts/app.js";
const NODE_NAME = "FunPackApplyLoraWeights";
const LORA_TYPES = ["general", "concept", "style", "quality", "character"];
const ADD_BUTTON_NAME = "+ Add LoRA";
function getLoraValues(nodeData) {
return nodeData?.input?.optional?.lora_0?.[0] || ["None"];
}
function getNextLoraIndex(node) {
let next = 0;
for (const widget of node.widgets || []) {
const match = /^lora_(\d+)$/.exec(widget.name || "");
if (match) {
next = Math.max(next, Number(match[1]) + 1);
}
}
return next;
}
function moveAddButtonToEnd(node) {
const button = (node.widgets || []).find((widget) => widget.name === ADD_BUTTON_NAME);
if (!button) {
return;
}
const index = node.widgets.indexOf(button);
if (index >= 0 && index !== node.widgets.length - 1) {
node.widgets.splice(index, 1);
node.widgets.push(button);
}
}
function resizeNode(node) {
const computed = node.computeSize();
node.size[0] = Math.max(node.size[0], computed[0]);
node.size[1] = computed[1];
}
function addLoraRow(node, nodeData, value = {}, markDirty = true) {
const index = getNextLoraIndex(node);
const loraWidget = node.addWidget(
"combo",
`lora_${index}`,
value.lora || "None",
undefined,
{ values: getLoraValues(nodeData) },
);
const typeWidget = node.addWidget(
"combo",
`lora_${index}_type`,
value.type || "general",
undefined,
{ values: LORA_TYPES },
);
const weightWidget = node.addWidget(
"number",
`lora_${index}_base_weight`,
value.base_weight ?? 1.0,
undefined,
{ min: -10.0, max: 10.0, step: 0.01, precision: 3 },
);
loraWidget.serialize = true;
typeWidget.serialize = true;
weightWidget.serialize = true;
moveAddButtonToEnd(node);
resizeNode(node);
if (markDirty) {
node.setDirtyCanvas(true, true);
}
}
function ensureAddButton(node, nodeData) {
if ((node.widgets || []).some((widget) => widget.name === ADD_BUTTON_NAME)) {
moveAddButtonToEnd(node);
return;
}
const button = node.addWidget("button", ADD_BUTTON_NAME, null, () => {
addLoraRow(node, nodeData);
});
button.serialize = false;
moveAddButtonToEnd(node);
resizeNode(node);
}
function restoreExtraRows(node, nodeData, info) {
const values = info?.widgets_values;
if (!Array.isArray(values)) {
return;
}
node.widgets = (node.widgets || []).filter((widget) => {
const match = /^lora_(\d+)(?:_type|_base_weight)?$/.exec(widget.name || "");
return !match || Number(match[1]) === 0;
});
const knownWidgetCount = 6;
for (let offset = knownWidgetCount; offset + 2 < values.length; offset += 3) {
addLoraRow(
node,
nodeData,
{
lora: values[offset],
type: values[offset + 1],
base_weight: values[offset + 2],
},
false,
);
}
}
app.registerExtension({
name: "funpack.loraWeights",
beforeRegisterNodeDef(nodeType, nodeData) {
if (nodeData.name !== NODE_NAME) {
return;
}
const originalOnNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function () {
originalOnNodeCreated?.apply(this, arguments);
ensureAddButton(this, nodeData);
};
const originalConfigure = nodeType.prototype.configure;
nodeType.prototype.configure = function (info) {
originalConfigure?.apply(this, arguments);
ensureAddButton(this, nodeData);
restoreExtraRows(this, nodeData, info);
};
},
});