nodes/silveroxides-ComfyUI-ModelUtils
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Add Markdown and CSV analysis reports

Browse Source
This commit is contained in:
silveroxides
2026-09-14 11:10:59 +02:00
parent e8eb2a0868
commit 7a606f8a02
3 changed files with 215 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/model_analysis.md
+9 -1
View File
@@ -13,10 +13,18 @@ in both modes.
## Outputs
`comparison_report` contains conventional numerical differences and topology
findings. `cwb_report` separately contains diagnostics derived from the
findings. Both reports use Markdown headings and tables. `cwb_report` separately contains diagnostics derived from the
similarity, alignment, and consensus stages of Consensus-Weighted Blending.
`documentation` contains this reference.
Two CSV-text outputs follow the existing three outputs, preserving their socket
positions: `layerwise_metrics_csv` and `layerwise_cwb_csv`. Connect them to a
text-saving node and use a `.csv` filename; analysis itself writes no files.
The CSV rows follow the same key order and displayed numeric precision as the
layerwise Markdown tables. Dtypes A/B, cosine mean/min/max, and A/B affinities
have separate columns. Undefined values remain `N/A`; an empty analysis produces
headers without data rows. Documentation-only mode returns empty CSV strings.
The reports begin with a global summary, followed by inferred blockwise values
and one entry for every comparable tensor. Block names are approximated from
the tensor-key hierarchy through its first numeric path component; they are not
nodes/model_analysis.py
+160 -125
View File
@@ -2,12 +2,14 @@
from __future__ import annotations
import csv
import gc
import heapq
import math
import re
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from io import StringIO
import comfy.utils
import folder_paths
@@ -580,14 +582,61 @@ def _average_layer_metrics(layer_stats: dict[str, RawStats]) -> dict[str, float
return result
def _metadata_lines(metadata_a: dict | None, metadata_b: dict | None) -> list[str]:
a = metadata_a or {}
b = metadata_b or {}
keys = sorted(set(a) | set(b))
differences = [key for key in keys if a.get(key) != b.get(key)]
lines = [f"Metadata keys: A={len(a)}, B={len(b)}, differing={len(differences)}"]
lines.extend(f"- {key}" for key in differences)
return lines
def _markdown_cell(value) -> str:
text = str(value).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
for character in ("\\", "|", "`", "*", "_", "[", "]"):
text = text.replace(character, "\\" + character)
return text.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "<br>")
def _markdown_table(headers, rows) -> str:
lines = [
"| " + " | ".join(_markdown_cell(value) for value in headers) + " |",
"| " + " | ".join("---" for _ in headers) + " |",
]
lines.extend("| " + " | ".join(_markdown_cell(value) for value in row) + " |" for row in rows)
return "\n".join(lines)
def _csv_table(headers, rows) -> str:
output = StringIO(newline="")
writer = csv.writer(output)
writer.writerow(headers)
writer.writerows(rows)
return output.getvalue()
def _comparison_layerwise(records):
headers = [
"key", "shape", "dtype_A", "dtype_B", "MAE", "RMSE", "max_abs",
"relative_L2", "cosine", "pearson", "exact", "sign", "norm_B/A",
]
rows = []
for record in sorted(records, key=lambda item: item.key):
metric = _metrics(record.raw)
rows.append([
record.key, _shape_text(record.shape), record.dtype_a, record.dtype_b,
*(_fmt(metric[name]) for name in (
"mae", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio",
)),
])
return headers, rows
def _cwb_table_data(entries, key_label="key"):
headers = [
key_label, "pair_cos_mean", "pair_cos_min", "pair_cos_max",
"mean_aff_A", "mean_aff_B", "median_aff_A", "median_aff_B",
"matched", "alignment", "gain",
]
rows = []
for key, stats in sorted(entries, key=lambda item: item[0]):
values = stats.values()
rows.append([key, *(_fmt(values[name]) for name in (
"pair_mean", "pair_min", "pair_max", "mean_aff_a", "mean_aff_b",
"median_aff_a", "median_aff_b", "matched", "alignment", "alignment_gain",
))])
return headers, rows
def _render_comparison(
@@ -606,86 +655,83 @@ def _render_comparison(
) -> str:
weighted = _metrics(global_raw)
layer_average = _average_layer_metrics(layer_stats)
lines = [
"MODEL COMPARISON SUMMARY",
f"Type: {model_type}",
f"Model A: {model_a}",
f"Model B: {model_b}",
f"Comparable floating tensors: {len(records)}",
f"Comparable parameters: {global_raw.total}",
f"A-only keys: {len(inventory['a_only'])}",
f"B-only keys: {len(inventory['b_only'])}",
f"Shape mismatches: {len(inventory['shape'])}",
f"Non-floating shared tensors: {len(inventory['nonfloat'])}",
f"Unsupported low-bit tensors: {len(inventory['low_bit'])}",
f"Excluded A tensors: {len(inventory['excluded_a'])}",
f"Excluded B tensors: {len(inventory['excluded_b'])}",
f"CUDA OOM CPU fallbacks: {len(fallback_keys)}",
"",
"PARAMETER-WEIGHTED GLOBAL METRICS",
sections = [
"# MODEL COMPARISON SUMMARY",
_markdown_table(["Field", "Value"], [
("Type", model_type), ("Model A", model_a), ("Model B", model_b),
("Comparable floating tensors", len(records)),
("Comparable parameters", global_raw.total),
("A-only keys", len(inventory["a_only"])),
("B-only keys", len(inventory["b_only"])),
("Shape mismatches", len(inventory["shape"])),
("Non-floating shared tensors", len(inventory["nonfloat"])),
("Unsupported low-bit tensors", len(inventory["low_bit"])),
("Excluded A tensors", len(inventory["excluded_a"])),
("Excluded B tensors", len(inventory["excluded_b"])),
("CUDA OOM CPU fallbacks", len(fallback_keys)),
]),
"## PARAMETER-WEIGHTED GLOBAL METRICS",
_markdown_table(["Metric", "Value"], [(name, _fmt(weighted[name])) for name in (
"mae", "mse", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", "coverage",
)]),
"## EQUAL-LAYER AVERAGES",
_markdown_table(["Metric", "Value"], [(name, _fmt(value)) for name, value in layer_average.items()]),
"## MOST DIVERGENT TENSORS",
]
for name in ("mae", "mse", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", "coverage"):
lines.append(f"{name}: {_fmt(weighted[name])}")
lines.extend(["", "EQUAL-LAYER AVERAGES"])
for name, value in layer_average.items():
lines.append(f"{name}: {_fmt(value)}")
metrics_by_record = [(record, _metrics(record.raw)) for record in records]
lines.extend(["", "MOST DIVERGENT TENSORS", "By relative L2:"])
for record, metric in sorted(metrics_by_record, key=lambda item: item[1]["relative_l2"] or -1.0, reverse=True)[:10]:
lines.append(f"- {record.key}: {_fmt(metric['relative_l2'])}")
lines.append("By MAE:")
for record, metric in sorted(metrics_by_record, key=lambda item: item[1]["mae"] or -1.0, reverse=True)[:10]:
lines.append(f"- {record.key}: {_fmt(metric['mae'])}")
lines.append("By lowest cosine:")
cosine_records = [(record, metric) for record, metric in metrics_by_record if metric["cosine"] is not None]
for record, metric in sorted(cosine_records, key=lambda item: item[1]["cosine"])[:10]:
lines.append(f"- {record.key}: {_fmt(metric['cosine'])}")
lines.extend(["", "INFERRED BLOCKWISE METRICS", "block | params | MAE | RMSE | relative_L2 | cosine"])
for title, metric_name, descending in (
("By relative L2", "relative_l2", True),
("By MAE", "mae", True),
("By lowest cosine", "cosine", False),
):
ranked = sorted(
((record, metric) for record, metric in metrics_by_record if metric[metric_name] is not None),
key=lambda item: item[1][metric_name], reverse=descending,
)[:10]
sections.extend([
"### " + title,
_markdown_table(["Rank", "Tensor", metric_name], [
(index, record.key, _fmt(metric[metric_name]))
for index, (record, metric) in enumerate(ranked, 1)
]),
])
block_rows = []
for block in sorted(block_stats):
raw = block_stats[block]
metric = _metrics(raw)
lines.append(
f"{block} | {raw.total} | {_fmt(metric['mae'])} | {_fmt(metric['rmse'])} | "
f"{_fmt(metric['relative_l2'])} | {_fmt(metric['cosine'])}"
)
lines.extend(["", "LAYERWISE TENSOR METRICS", "key | shape | dtype A/B | MAE | RMSE | max_abs | relative_L2 | cosine | pearson | exact | sign | norm_B/A"])
for record, metric in sorted(metrics_by_record, key=lambda item: item[0].key):
lines.append(
f"{record.key} | {_shape_text(record.shape)} | {record.dtype_a}/{record.dtype_b} | "
f"{_fmt(metric['mae'])} | {_fmt(metric['rmse'])} | {_fmt(metric['max'])} | "
f"{_fmt(metric['relative_l2'])} | {_fmt(metric['cosine'])} | {_fmt(metric['pearson'])} | "
f"{_fmt(metric['exact'])} | {_fmt(metric['sign'])} | {_fmt(metric['norm_ratio'])}"
)
lines.extend(["", "TOP SCALAR WEIGHT DIFFERENCES"])
for value, key, flat_index, a_value, b_value, shape in sorted(top_weights, reverse=True):
lines.append(
f"- {key}{_coordinate(shape, flat_index)}: |delta|={_fmt(value)}, "
f"A={_fmt(a_value)}, B={_fmt(b_value)}"
)
if not top_weights:
lines.append("- None")
lines.extend(["", "TOPOLOGY AND DATA FINDINGS"])
block_rows.append([block, raw.total, *(_fmt(metric[name]) for name in ("mae", "rmse", "relative_l2", "cosine"))])
sections.extend([
"## INFERRED BLOCKWISE METRICS",
_markdown_table(["block", "params", "MAE", "RMSE", "relative_L2", "cosine"], block_rows),
"## LAYERWISE TENSOR METRICS",
_markdown_table(*_comparison_layerwise(records)),
"## TOP SCALAR WEIGHT DIFFERENCES",
_markdown_table(["Rank", "Tensor", "Coordinate", "|delta|", "A", "B"], [
(index, key, _coordinate(shape, flat_index), _fmt(value), _fmt(a_value), _fmt(b_value))
for index, (value, key, flat_index, a_value, b_value, shape) in enumerate(sorted(top_weights, reverse=True), 1)
]) if top_weights else "None.",
"## TOPOLOGY AND DATA FINDINGS",
])
for label, name in (
("A ONLY", "a_only"), ("B ONLY", "b_only"), ("SHAPE MISMATCH", "shape"),
("NON-FLOATING", "nonfloat"), ("LOW-BIT UNSUPPORTED", "low_bit"),
("EXCLUDED A", "excluded_a"), ("EXCLUDED B", "excluded_b"),
):
lines.append(f"[{label}] ({len(inventory[name])})")
lines.extend(f"- {value}" for value in inventory[name])
if not inventory[name]:
lines.append("- None")
lines.append(f"[CUDA OOM -> CPU] ({len(fallback_keys)})")
lines.extend(f"- {key}" for key in fallback_keys)
if not fallback_keys:
lines.append("- None")
lines.extend(["", "METADATA"])
lines.extend(_metadata_lines(metadata_a, metadata_b))
return "\n".join(lines)
sections.extend([
f"### [{label}] ({len(inventory[name])})",
_markdown_table(["Key / finding"], [(value,) for value in inventory[name]]) if inventory[name] else "None.",
])
sections.extend([
f"### [CUDA OOM -> CPU] ({len(fallback_keys)})",
_markdown_table(["Key"], [(key,) for key in fallback_keys]) if fallback_keys else "None.",
"## METADATA",
])
a, b = metadata_a or {}, metadata_b or {}
differences = [key for key in sorted(set(a) | set(b)) if a.get(key) != b.get(key)]
sections.append(_markdown_table(["Metadata keys A", "Metadata keys B", "Differing keys"], [(len(a), len(b), len(differences))]))
if differences:
sections.append(_markdown_table(["Differing metadata key"], [(key,) for key in differences]))
return "\n\n".join(sections) + "\n"
def _render_cwb(
@@ -700,49 +746,32 @@ def _render_cwb(
for record in records:
global_cwb.add(record.cwb)
values = global_cwb.values()
lines = [
"CWB-DERIVED SIMILARITY SUMMARY",
f"Model A: {model_a}",
f"Model B: {model_b}",
f"Alignment scope: {alignment_scope}",
"Boundary: CWB similarity, alignment, and consensus diagnostics only.",
"Excluded: merge weighting, contribution calculation, vector reconstitution, norm rescaling, and output writing.",
"",
"GLOBAL CWB-DERIVED DIAGNOSTICS",
f"Pairwise row cosine mean/min/max: {_fmt(values['pair_mean'])} / {_fmt(values['pair_min'])} / {_fmt(values['pair_max'])}",
f"Mean-consensus affinity A/B: {_fmt(values['mean_aff_a'])} / {_fmt(values['mean_aff_b'])}",
f"Median-consensus affinity A/B: {_fmt(values['median_aff_a'])} / {_fmt(values['median_aff_b'])}",
f"Greedy alignment coverage: {_fmt(values['matched'])}",
f"Greedy matched cosine mean/min/max: {_fmt(values['alignment'])} / {_fmt(values['alignment_min'])} / {_fmt(values['alignment_max'])}",
f"Greedy improvement over index alignment: {_fmt(values['alignment_gain'])}",
"",
"INFERRED BLOCKWISE CWB DIAGNOSTICS",
"block | pair_cos mean/min/max | mean_aff_A/B | median_aff_A/B | matched | alignment | gain",
sections = [
"# CWB-DERIVED SIMILARITY SUMMARY",
_markdown_table(["Field", "Value"], [
("Model A", model_a), ("Model B", model_b), ("Alignment scope", alignment_scope),
("Boundary", "CWB similarity, alignment, and consensus diagnostics only."),
("Excluded", "merge weighting, contribution calculation, vector reconstitution, norm rescaling, and output writing."),
]),
"## GLOBAL CWB-DERIVED DIAGNOSTICS",
_markdown_table(["Metric", "Value"], [
("Pairwise row cosine mean/min/max", " / ".join(_fmt(values[name]) for name in ("pair_mean", "pair_min", "pair_max"))),
("Mean-consensus affinity A/B", " / ".join(_fmt(values[name]) for name in ("mean_aff_a", "mean_aff_b"))),
("Median-consensus affinity A/B", " / ".join(_fmt(values[name]) for name in ("median_aff_a", "median_aff_b"))),
("Greedy alignment coverage", _fmt(values["matched"])),
("Greedy matched cosine mean/min/max", " / ".join(_fmt(values[name]) for name in ("alignment", "alignment_min", "alignment_max"))),
("Greedy improvement over index alignment", _fmt(values["alignment_gain"])),
]),
"## INFERRED BLOCKWISE CWB DIAGNOSTICS",
_markdown_table(*_cwb_table_data(block_cwb.items(), "block")),
"## LAYERWISE CWB DIAGNOSTICS",
_markdown_table(*_cwb_table_data((record.key, record.cwb) for record in records)),
"## CUDA OOM CPU FALLBACKS",
f"Count: {len(fallback_keys)}",
]
for block in sorted(block_cwb):
entry = block_cwb[block].values()
lines.append(
f"{block} | {_fmt(entry['pair_mean'])}/{_fmt(entry['pair_min'])}/{_fmt(entry['pair_max'])} | "
f"{_fmt(entry['mean_aff_a'])}/{_fmt(entry['mean_aff_b'])} | "
f"{_fmt(entry['median_aff_a'])}/{_fmt(entry['median_aff_b'])} | "
f"{_fmt(entry['matched'])} | {_fmt(entry['alignment'])} | {_fmt(entry['alignment_gain'])}"
)
lines.extend([
"", "LAYERWISE CWB DIAGNOSTICS",
"key | pair_cos mean/min/max | mean_aff_A/B | median_aff_A/B | matched | alignment | gain",
])
for record in sorted(records, key=lambda value: value.key):
entry = record.cwb.values()
lines.append(
f"{record.key} | {_fmt(entry['pair_mean'])}/{_fmt(entry['pair_min'])}/{_fmt(entry['pair_max'])} | "
f"{_fmt(entry['mean_aff_a'])}/{_fmt(entry['mean_aff_b'])} | "
f"{_fmt(entry['median_aff_a'])}/{_fmt(entry['median_aff_b'])} | "
f"{_fmt(entry['matched'])} | {_fmt(entry['alignment'])} | {_fmt(entry['alignment_gain'])}"
)
lines.append("")
lines.append(f"CUDA OOM CPU fallbacks: {len(fallback_keys)}")
lines.extend(f"- {key}" for key in fallback_keys)
return "\n".join(lines)
if fallback_keys:
sections.append(_markdown_table(["Key"], [(key,) for key in fallback_keys]))
return "\n\n".join(sections) + "\n"
class ModelAnalysisLogic:
@@ -756,7 +785,7 @@ class ModelAnalysisLogic:
*,
embedding_alignment: bool = False,
lora_mode: bool = False,
) -> tuple[str, str]:
) -> tuple[str, str, str, str]:
paths = []
for name in (model_a, model_b):
path = folder_paths.get_full_path(model_type, name)
@@ -1037,7 +1066,9 @@ class ModelAnalysisLogic:
if embedding_alignment else "fixed tensor rows (index only)"
),
)
return comparison, cwb_report
layerwise_metrics_csv = _csv_table(*_comparison_layerwise(records))
layerwise_cwb_csv = _csv_table(*_cwb_table_data((record.key, record.cwb) for record in cwb_records))
return comparison, cwb_report, layerwise_metrics_csv, layerwise_cwb_csv
finally:
for handler in handlers:
handler.__exit__(None, None, None)
@@ -1122,7 +1153,7 @@ class _ModelAnalysisNode(io.ComfyNode):
node_id=cls.NODE_ID,
display_name=cls.DISPLAY_NAME,
category="ModelUtils/Analysis",
description="Compare two model files without merging them and return separate standard-metric and CWB-diagnostic reports.",
description="Compare two model files without merging them. Return Markdown reports and separate layerwise CSV text.",
inputs=_common_inputs(
cls.MODEL_TYPE,
alignment_control=cls.EMBEDDING_ALIGNMENT or cls.LORA_MODE,
@@ -1131,6 +1162,8 @@ class _ModelAnalysisNode(io.ComfyNode):
io.String.Output(display_name="comparison_report"),
io.String.Output(display_name="cwb_report"),
io.String.Output(display_name="documentation"),
io.String.Output(display_name="layerwise_metrics_csv", tooltip="CSV text for every comparable tensor; save as .csv with a text-saving node."),
io.String.Output(display_name="layerwise_cwb_csv", tooltip="CSV text for layerwise CWB diagnostics, with grouped metrics in separate columns."),
],
is_experimental=True,
)
@@ -1143,13 +1176,15 @@ class _ModelAnalysisNode(io.ComfyNode):
"Documentation mode active. No comparison performed.",
"Documentation mode active. No CWB analysis performed.",
documentation,
"",
"",
)
comparison, cwb_report = ModelAnalysisLogic.execute(
comparison, cwb_report, metrics_csv, cwb_csv = ModelAnalysisLogic.execute(
kwargs["model_a"], kwargs["model_b"], cls.MODEL_TYPE, kwargs,
embedding_alignment=cls.EMBEDDING_ALIGNMENT,
lora_mode=cls.LORA_MODE,
)
return io.NodeOutput(comparison, cwb_report, documentation)
return io.NodeOutput(comparison, cwb_report, documentation, metrics_csv, cwb_csv)
class CheckpointModelAnalysis(_ModelAnalysisNode):
tests/test_model_analysis.py
+46 -20
View File
@@ -1,4 +1,6 @@
import importlib
import csv
from io import StringIO
import math
import sys
import types
@@ -67,15 +69,17 @@ def test_five_node_schema_contract(analysis):
} & set(input_ids)
assert [item.display_name for item in schema.outputs] == [
"comparison_report", "cwb_report", "documentation",
"layerwise_metrics_csv", "layerwise_cwb_csv",
]
def test_documentation_mode_has_three_separate_outputs(analysis):
def test_documentation_mode_preserves_reports_and_returns_empty_csv(analysis):
result = analysis.DiffusionModelAnalysis.execute(execution_mode="DOCUMENTATION ONLY")
assert len(result.result) == 3
assert len(result.result) == 5
assert "No comparison performed" in result[0]
assert "No CWB analysis performed" in result[1]
assert result[2].startswith("# Two-Model Similarity Analysis")
assert result[3] == result[4] == ""
def test_raw_metrics_have_independent_expected_values(analysis):
@@ -125,16 +129,17 @@ def test_zero_constant_and_nonfinite_metrics_are_explicit(analysis):
def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, analysis):
tensor_key = 'model.blocks.0.weight|"quoted",\nname'
path_a = tmp_path / "a.safetensors"
path_b = tmp_path / "b.safetensors"
save_file({
"model.blocks.0.weight": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
tensor_key: torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
"only_a": torch.ones(1),
"shape": torch.ones(2),
"integer": torch.tensor([1, 2], dtype=torch.int64),
}, str(path_a), metadata={"source": "a"})
save_file({
"model.blocks.0.weight": torch.tensor([[1.0, 0.0], [5.0, 4.0]]),
tensor_key: torch.tensor([[1.0, 0.0], [5.0, 4.0]]),
"only_b": torch.ones(1),
"shape": torch.ones(3),
"integer": torch.tensor([1, 3], dtype=torch.int64),
@@ -159,12 +164,29 @@ def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, a
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "mark_processed", track_processed)
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "async_stream", track_async_stream)
comparison, cwb_report = analysis.ModelAnalysisLogic.execute(
"a", "b", "diffusion_models", _params()
result = analysis.DiffusionModelAnalysis.execute(
model_a="a", model_b="b", execution_mode="ANALYZE", **_params()
)
assert "Comparable floating tensors: 1" in comparison
assert "mae: 1" in comparison
assert "mse: 2" in comparison
comparison, cwb_report, documentation, metrics_csv, cwb_csv = result.result
assert documentation.startswith("# Two-Model Similarity Analysis")
assert comparison.startswith("# MODEL COMPARISON SUMMARY\n\n")
assert "| Comparable floating tensors | 1 |" in comparison
assert "| mae | 1 |" in comparison
assert "| mse | 2 |" in comparison
assert '| model.blocks.0.weight\\|"quoted",<br>name |' in comparison
assert "| --- | --- |" in comparison
metrics_rows = list(csv.DictReader(StringIO(metrics_csv)))
cwb_rows = list(csv.DictReader(StringIO(cwb_csv)))
assert len(metrics_rows) == len(cwb_rows) == 1
assert metrics_rows[0]["key"] == cwb_rows[0]["key"] == tensor_key
assert float(metrics_rows[0]["MAE"]) == 1
assert float(metrics_rows[0]["RMSE"]) == pytest.approx(math.sqrt(2), rel=1e-5)
assert metrics_rows[0]["dtype_A"] == metrics_rows[0]["dtype_B"] == "torch.float32"
assert cwb_rows[0]["matched"] == "N/A"
assert float(cwb_rows[0]["pair_cos_min"]) <= float(cwb_rows[0]["pair_cos_mean"]) <= float(cwb_rows[0]["pair_cos_max"])
top_rows = comparison.split("## TOP SCALAR WEIGHT DIFFERENCES\n\n", 1)[1].split("\n\n", 1)[0].splitlines()[2:]
assert len(top_rows) == 3
assert [float(row.rsplit(" | ", 4)[2]) for row in top_rows] == [2, 2, 0]
assert "[A ONLY] (1)" in comparison
assert "[B ONLY] (1)" in comparison
assert "[SHAPE MISMATCH] (1)" in comparison
@@ -175,7 +197,7 @@ def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, a
assert "PRESET-IMPLIED CONTRIBUTIONS" not in cwb_report
assert "Average contribution" not in cwb_report
assert len(processed) == 4
assert {key for _, key in processed} == {"model.blocks.0.weight", "integer"}
assert {key for _, key in processed} == {tensor_key, "integer"}
assert len(async_calls) == 4
assert all(len(call[1]) == 1 for call in async_calls)
assert all(call[2]["batch_size"] == 1 for call in async_calls)
@@ -214,15 +236,17 @@ def test_lora_cross_format_keys_compare_as_one_logical_pair(monkeypatch, tmp_pat
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "mark_processed", track_processed)
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "async_stream", track_async_stream)
comparison, cwb_report = analysis.ModelAnalysisLogic.execute(
comparison, cwb_report, metrics_csv, cwb_csv = analysis.ModelAnalysisLogic.execute(
"a", "b", "loras", _params(), lora_mode=True,
)
assert "Comparable floating tensors: 2" in comparison
assert "| Comparable floating tensors | 2 |" in comparison
assert "[A ONLY] (0)" in comparison
assert "[B ONLY] (0)" in comparison
assert "foo.down |" in comparison
assert "foo.up |" in comparison
assert "foo.[down+up] |" in cwb_report
assert "foo.\\[down+up\\] |" in cwb_report
assert [row["key"] for row in csv.DictReader(StringIO(metrics_csv))] == ["foo.down", "foo.up"]
assert [row["key"] for row in csv.DictReader(StringIO(cwb_csv))] == ["foo.[down+up]"]
assert len(processed) == 4
assert len({entry for entry in processed}) == 4
assert len(async_calls) == 2
@@ -254,21 +278,23 @@ def test_excluded_lora_blocks_do_not_enter_metrics(
monkeypatch.setattr(analysis, "prepare_for_large_operation", lambda *args, **kwargs: None)
monkeypatch.setattr(analysis, "cleanup_after_operation", lambda: None)
comparison, cwb_report = analysis.ModelAnalysisLogic.execute(
comparison, cwb_report, metrics_csv, cwb_csv = analysis.ModelAnalysisLogic.execute(
"a", "b", "loras",
_params(exclude_patterns=patterns, glob_patterns=glob_patterns, include_mode=include_mode),
lora_mode=True,
)
assert "Comparable floating tensors: 2" in comparison
assert "Excluded A tensors: 2" in comparison
assert "Excluded B tensors: 2" in comparison
assert "| Comparable floating tensors | 2 |" in comparison
assert "| Excluded A tensors | 2 |" in comparison
assert "| Excluded B tensors | 2 |" in comparison
layerwise = comparison.split("LAYERWISE TENSOR METRICS", 1)[1].split(
"TOP SCALAR WEIGHT DIFFERENCES", 1
)[0]
assert "blocks_3_" in layerwise
assert "blocks_4_" not in layerwise
assert "blocks_4_" not in cwb_report
assert "blocks\\_3\\_" in layerwise
assert "blocks\\_4\\_" not in layerwise
assert "blocks\\_4\\_" not in cwb_report
assert all("blocks_3_" in row["key"] for row in csv.DictReader(StringIO(metrics_csv)))
assert all("blocks_3_" in row["key"] for row in csv.DictReader(StringIO(cwb_csv)))
@pytest.mark.parametrize("patterns, glob_patterns, expected_keys", [