diff --git a/docs/model_analysis.md b/docs/model_analysis.md
index 116dbc2..c5a6552 100644
--- a/docs/model_analysis.md
+++ b/docs/model_analysis.md
@@ -13,10 +13,18 @@ in both modes.
## Outputs
`comparison_report` contains conventional numerical differences and topology
-findings. `cwb_report` separately contains diagnostics derived from the
+findings. Both reports use Markdown headings and tables. `cwb_report` separately contains diagnostics derived from the
similarity, alignment, and consensus stages of Consensus-Weighted Blending.
`documentation` contains this reference.
+Two CSV-text outputs follow the existing three outputs, preserving their socket
+positions: `layerwise_metrics_csv` and `layerwise_cwb_csv`. Connect them to a
+text-saving node and use a `.csv` filename; analysis itself writes no files.
+The CSV rows follow the same key order and displayed numeric precision as the
+layerwise Markdown tables. Dtypes A/B, cosine mean/min/max, and A/B affinities
+have separate columns. Undefined values remain `N/A`; an empty analysis produces
+headers without data rows. Documentation-only mode returns empty CSV strings.
+
The reports begin with a global summary, followed by inferred blockwise values
and one entry for every comparable tensor. Block names are approximated from
the tensor-key hierarchy through its first numeric path component; they are not
diff --git a/nodes/model_analysis.py b/nodes/model_analysis.py
index 02a7ef6..4e3f460 100644
--- a/nodes/model_analysis.py
+++ b/nodes/model_analysis.py
@@ -2,12 +2,14 @@
from __future__ import annotations
+import csv
import gc
import heapq
import math
import re
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
+from io import StringIO
import comfy.utils
import folder_paths
@@ -580,14 +582,61 @@ def _average_layer_metrics(layer_stats: dict[str, RawStats]) -> dict[str, float
return result
-def _metadata_lines(metadata_a: dict | None, metadata_b: dict | None) -> list[str]:
- a = metadata_a or {}
- b = metadata_b or {}
- keys = sorted(set(a) | set(b))
- differences = [key for key in keys if a.get(key) != b.get(key)]
- lines = [f"Metadata keys: A={len(a)}, B={len(b)}, differing={len(differences)}"]
- lines.extend(f"- {key}" for key in differences)
- return lines
+def _markdown_cell(value) -> str:
+ text = str(value).replace("&", "&").replace("<", "<").replace(">", ">")
+ for character in ("\\", "|", "`", "*", "_", "[", "]"):
+ text = text.replace(character, "\\" + character)
+ return text.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "
")
+
+
+def _markdown_table(headers, rows) -> str:
+ lines = [
+ "| " + " | ".join(_markdown_cell(value) for value in headers) + " |",
+ "| " + " | ".join("---" for _ in headers) + " |",
+ ]
+ lines.extend("| " + " | ".join(_markdown_cell(value) for value in row) + " |" for row in rows)
+ return "\n".join(lines)
+
+
+def _csv_table(headers, rows) -> str:
+ output = StringIO(newline="")
+ writer = csv.writer(output)
+ writer.writerow(headers)
+ writer.writerows(rows)
+ return output.getvalue()
+
+
+def _comparison_layerwise(records):
+ headers = [
+ "key", "shape", "dtype_A", "dtype_B", "MAE", "RMSE", "max_abs",
+ "relative_L2", "cosine", "pearson", "exact", "sign", "norm_B/A",
+ ]
+ rows = []
+ for record in sorted(records, key=lambda item: item.key):
+ metric = _metrics(record.raw)
+ rows.append([
+ record.key, _shape_text(record.shape), record.dtype_a, record.dtype_b,
+ *(_fmt(metric[name]) for name in (
+ "mae", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio",
+ )),
+ ])
+ return headers, rows
+
+
+def _cwb_table_data(entries, key_label="key"):
+ headers = [
+ key_label, "pair_cos_mean", "pair_cos_min", "pair_cos_max",
+ "mean_aff_A", "mean_aff_B", "median_aff_A", "median_aff_B",
+ "matched", "alignment", "gain",
+ ]
+ rows = []
+ for key, stats in sorted(entries, key=lambda item: item[0]):
+ values = stats.values()
+ rows.append([key, *(_fmt(values[name]) for name in (
+ "pair_mean", "pair_min", "pair_max", "mean_aff_a", "mean_aff_b",
+ "median_aff_a", "median_aff_b", "matched", "alignment", "alignment_gain",
+ ))])
+ return headers, rows
def _render_comparison(
@@ -606,86 +655,83 @@ def _render_comparison(
) -> str:
weighted = _metrics(global_raw)
layer_average = _average_layer_metrics(layer_stats)
- lines = [
- "MODEL COMPARISON SUMMARY",
- f"Type: {model_type}",
- f"Model A: {model_a}",
- f"Model B: {model_b}",
- f"Comparable floating tensors: {len(records)}",
- f"Comparable parameters: {global_raw.total}",
- f"A-only keys: {len(inventory['a_only'])}",
- f"B-only keys: {len(inventory['b_only'])}",
- f"Shape mismatches: {len(inventory['shape'])}",
- f"Non-floating shared tensors: {len(inventory['nonfloat'])}",
- f"Unsupported low-bit tensors: {len(inventory['low_bit'])}",
- f"Excluded A tensors: {len(inventory['excluded_a'])}",
- f"Excluded B tensors: {len(inventory['excluded_b'])}",
- f"CUDA OOM CPU fallbacks: {len(fallback_keys)}",
- "",
- "PARAMETER-WEIGHTED GLOBAL METRICS",
+ sections = [
+ "# MODEL COMPARISON SUMMARY",
+ _markdown_table(["Field", "Value"], [
+ ("Type", model_type), ("Model A", model_a), ("Model B", model_b),
+ ("Comparable floating tensors", len(records)),
+ ("Comparable parameters", global_raw.total),
+ ("A-only keys", len(inventory["a_only"])),
+ ("B-only keys", len(inventory["b_only"])),
+ ("Shape mismatches", len(inventory["shape"])),
+ ("Non-floating shared tensors", len(inventory["nonfloat"])),
+ ("Unsupported low-bit tensors", len(inventory["low_bit"])),
+ ("Excluded A tensors", len(inventory["excluded_a"])),
+ ("Excluded B tensors", len(inventory["excluded_b"])),
+ ("CUDA OOM CPU fallbacks", len(fallback_keys)),
+ ]),
+ "## PARAMETER-WEIGHTED GLOBAL METRICS",
+ _markdown_table(["Metric", "Value"], [(name, _fmt(weighted[name])) for name in (
+ "mae", "mse", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", "coverage",
+ )]),
+ "## EQUAL-LAYER AVERAGES",
+ _markdown_table(["Metric", "Value"], [(name, _fmt(value)) for name, value in layer_average.items()]),
+ "## MOST DIVERGENT TENSORS",
]
- for name in ("mae", "mse", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", "coverage"):
- lines.append(f"{name}: {_fmt(weighted[name])}")
- lines.extend(["", "EQUAL-LAYER AVERAGES"])
- for name, value in layer_average.items():
- lines.append(f"{name}: {_fmt(value)}")
-
metrics_by_record = [(record, _metrics(record.raw)) for record in records]
- lines.extend(["", "MOST DIVERGENT TENSORS", "By relative L2:"])
- for record, metric in sorted(metrics_by_record, key=lambda item: item[1]["relative_l2"] or -1.0, reverse=True)[:10]:
- lines.append(f"- {record.key}: {_fmt(metric['relative_l2'])}")
- lines.append("By MAE:")
- for record, metric in sorted(metrics_by_record, key=lambda item: item[1]["mae"] or -1.0, reverse=True)[:10]:
- lines.append(f"- {record.key}: {_fmt(metric['mae'])}")
- lines.append("By lowest cosine:")
- cosine_records = [(record, metric) for record, metric in metrics_by_record if metric["cosine"] is not None]
- for record, metric in sorted(cosine_records, key=lambda item: item[1]["cosine"])[:10]:
- lines.append(f"- {record.key}: {_fmt(metric['cosine'])}")
-
- lines.extend(["", "INFERRED BLOCKWISE METRICS", "block | params | MAE | RMSE | relative_L2 | cosine"])
+ for title, metric_name, descending in (
+ ("By relative L2", "relative_l2", True),
+ ("By MAE", "mae", True),
+ ("By lowest cosine", "cosine", False),
+ ):
+ ranked = sorted(
+ ((record, metric) for record, metric in metrics_by_record if metric[metric_name] is not None),
+ key=lambda item: item[1][metric_name], reverse=descending,
+ )[:10]
+ sections.extend([
+ "### " + title,
+ _markdown_table(["Rank", "Tensor", metric_name], [
+ (index, record.key, _fmt(metric[metric_name]))
+ for index, (record, metric) in enumerate(ranked, 1)
+ ]),
+ ])
+ block_rows = []
for block in sorted(block_stats):
raw = block_stats[block]
metric = _metrics(raw)
- lines.append(
- f"{block} | {raw.total} | {_fmt(metric['mae'])} | {_fmt(metric['rmse'])} | "
- f"{_fmt(metric['relative_l2'])} | {_fmt(metric['cosine'])}"
- )
-
- lines.extend(["", "LAYERWISE TENSOR METRICS", "key | shape | dtype A/B | MAE | RMSE | max_abs | relative_L2 | cosine | pearson | exact | sign | norm_B/A"])
- for record, metric in sorted(metrics_by_record, key=lambda item: item[0].key):
- lines.append(
- f"{record.key} | {_shape_text(record.shape)} | {record.dtype_a}/{record.dtype_b} | "
- f"{_fmt(metric['mae'])} | {_fmt(metric['rmse'])} | {_fmt(metric['max'])} | "
- f"{_fmt(metric['relative_l2'])} | {_fmt(metric['cosine'])} | {_fmt(metric['pearson'])} | "
- f"{_fmt(metric['exact'])} | {_fmt(metric['sign'])} | {_fmt(metric['norm_ratio'])}"
- )
-
- lines.extend(["", "TOP SCALAR WEIGHT DIFFERENCES"])
- for value, key, flat_index, a_value, b_value, shape in sorted(top_weights, reverse=True):
- lines.append(
- f"- {key}{_coordinate(shape, flat_index)}: |delta|={_fmt(value)}, "
- f"A={_fmt(a_value)}, B={_fmt(b_value)}"
- )
- if not top_weights:
- lines.append("- None")
-
- lines.extend(["", "TOPOLOGY AND DATA FINDINGS"])
+ block_rows.append([block, raw.total, *(_fmt(metric[name]) for name in ("mae", "rmse", "relative_l2", "cosine"))])
+ sections.extend([
+ "## INFERRED BLOCKWISE METRICS",
+ _markdown_table(["block", "params", "MAE", "RMSE", "relative_L2", "cosine"], block_rows),
+ "## LAYERWISE TENSOR METRICS",
+ _markdown_table(*_comparison_layerwise(records)),
+ "## TOP SCALAR WEIGHT DIFFERENCES",
+ _markdown_table(["Rank", "Tensor", "Coordinate", "|delta|", "A", "B"], [
+ (index, key, _coordinate(shape, flat_index), _fmt(value), _fmt(a_value), _fmt(b_value))
+ for index, (value, key, flat_index, a_value, b_value, shape) in enumerate(sorted(top_weights, reverse=True), 1)
+ ]) if top_weights else "None.",
+ "## TOPOLOGY AND DATA FINDINGS",
+ ])
for label, name in (
("A ONLY", "a_only"), ("B ONLY", "b_only"), ("SHAPE MISMATCH", "shape"),
("NON-FLOATING", "nonfloat"), ("LOW-BIT UNSUPPORTED", "low_bit"),
("EXCLUDED A", "excluded_a"), ("EXCLUDED B", "excluded_b"),
):
- lines.append(f"[{label}] ({len(inventory[name])})")
- lines.extend(f"- {value}" for value in inventory[name])
- if not inventory[name]:
- lines.append("- None")
- lines.append(f"[CUDA OOM -> CPU] ({len(fallback_keys)})")
- lines.extend(f"- {key}" for key in fallback_keys)
- if not fallback_keys:
- lines.append("- None")
- lines.extend(["", "METADATA"])
- lines.extend(_metadata_lines(metadata_a, metadata_b))
- return "\n".join(lines)
+ sections.extend([
+ f"### [{label}] ({len(inventory[name])})",
+ _markdown_table(["Key / finding"], [(value,) for value in inventory[name]]) if inventory[name] else "None.",
+ ])
+ sections.extend([
+ f"### [CUDA OOM -> CPU] ({len(fallback_keys)})",
+ _markdown_table(["Key"], [(key,) for key in fallback_keys]) if fallback_keys else "None.",
+ "## METADATA",
+ ])
+ a, b = metadata_a or {}, metadata_b or {}
+ differences = [key for key in sorted(set(a) | set(b)) if a.get(key) != b.get(key)]
+ sections.append(_markdown_table(["Metadata keys A", "Metadata keys B", "Differing keys"], [(len(a), len(b), len(differences))]))
+ if differences:
+ sections.append(_markdown_table(["Differing metadata key"], [(key,) for key in differences]))
+ return "\n\n".join(sections) + "\n"
def _render_cwb(
@@ -700,49 +746,32 @@ def _render_cwb(
for record in records:
global_cwb.add(record.cwb)
values = global_cwb.values()
- lines = [
- "CWB-DERIVED SIMILARITY SUMMARY",
- f"Model A: {model_a}",
- f"Model B: {model_b}",
- f"Alignment scope: {alignment_scope}",
- "Boundary: CWB similarity, alignment, and consensus diagnostics only.",
- "Excluded: merge weighting, contribution calculation, vector reconstitution, norm rescaling, and output writing.",
- "",
- "GLOBAL CWB-DERIVED DIAGNOSTICS",
- f"Pairwise row cosine mean/min/max: {_fmt(values['pair_mean'])} / {_fmt(values['pair_min'])} / {_fmt(values['pair_max'])}",
- f"Mean-consensus affinity A/B: {_fmt(values['mean_aff_a'])} / {_fmt(values['mean_aff_b'])}",
- f"Median-consensus affinity A/B: {_fmt(values['median_aff_a'])} / {_fmt(values['median_aff_b'])}",
- f"Greedy alignment coverage: {_fmt(values['matched'])}",
- f"Greedy matched cosine mean/min/max: {_fmt(values['alignment'])} / {_fmt(values['alignment_min'])} / {_fmt(values['alignment_max'])}",
- f"Greedy improvement over index alignment: {_fmt(values['alignment_gain'])}",
- "",
- "INFERRED BLOCKWISE CWB DIAGNOSTICS",
- "block | pair_cos mean/min/max | mean_aff_A/B | median_aff_A/B | matched | alignment | gain",
+ sections = [
+ "# CWB-DERIVED SIMILARITY SUMMARY",
+ _markdown_table(["Field", "Value"], [
+ ("Model A", model_a), ("Model B", model_b), ("Alignment scope", alignment_scope),
+ ("Boundary", "CWB similarity, alignment, and consensus diagnostics only."),
+ ("Excluded", "merge weighting, contribution calculation, vector reconstitution, norm rescaling, and output writing."),
+ ]),
+ "## GLOBAL CWB-DERIVED DIAGNOSTICS",
+ _markdown_table(["Metric", "Value"], [
+ ("Pairwise row cosine mean/min/max", " / ".join(_fmt(values[name]) for name in ("pair_mean", "pair_min", "pair_max"))),
+ ("Mean-consensus affinity A/B", " / ".join(_fmt(values[name]) for name in ("mean_aff_a", "mean_aff_b"))),
+ ("Median-consensus affinity A/B", " / ".join(_fmt(values[name]) for name in ("median_aff_a", "median_aff_b"))),
+ ("Greedy alignment coverage", _fmt(values["matched"])),
+ ("Greedy matched cosine mean/min/max", " / ".join(_fmt(values[name]) for name in ("alignment", "alignment_min", "alignment_max"))),
+ ("Greedy improvement over index alignment", _fmt(values["alignment_gain"])),
+ ]),
+ "## INFERRED BLOCKWISE CWB DIAGNOSTICS",
+ _markdown_table(*_cwb_table_data(block_cwb.items(), "block")),
+ "## LAYERWISE CWB DIAGNOSTICS",
+ _markdown_table(*_cwb_table_data((record.key, record.cwb) for record in records)),
+ "## CUDA OOM CPU FALLBACKS",
+ f"Count: {len(fallback_keys)}",
]
- for block in sorted(block_cwb):
- entry = block_cwb[block].values()
- lines.append(
- f"{block} | {_fmt(entry['pair_mean'])}/{_fmt(entry['pair_min'])}/{_fmt(entry['pair_max'])} | "
- f"{_fmt(entry['mean_aff_a'])}/{_fmt(entry['mean_aff_b'])} | "
- f"{_fmt(entry['median_aff_a'])}/{_fmt(entry['median_aff_b'])} | "
- f"{_fmt(entry['matched'])} | {_fmt(entry['alignment'])} | {_fmt(entry['alignment_gain'])}"
- )
- lines.extend([
- "", "LAYERWISE CWB DIAGNOSTICS",
- "key | pair_cos mean/min/max | mean_aff_A/B | median_aff_A/B | matched | alignment | gain",
- ])
- for record in sorted(records, key=lambda value: value.key):
- entry = record.cwb.values()
- lines.append(
- f"{record.key} | {_fmt(entry['pair_mean'])}/{_fmt(entry['pair_min'])}/{_fmt(entry['pair_max'])} | "
- f"{_fmt(entry['mean_aff_a'])}/{_fmt(entry['mean_aff_b'])} | "
- f"{_fmt(entry['median_aff_a'])}/{_fmt(entry['median_aff_b'])} | "
- f"{_fmt(entry['matched'])} | {_fmt(entry['alignment'])} | {_fmt(entry['alignment_gain'])}"
- )
- lines.append("")
- lines.append(f"CUDA OOM CPU fallbacks: {len(fallback_keys)}")
- lines.extend(f"- {key}" for key in fallback_keys)
- return "\n".join(lines)
+ if fallback_keys:
+ sections.append(_markdown_table(["Key"], [(key,) for key in fallback_keys]))
+ return "\n\n".join(sections) + "\n"
class ModelAnalysisLogic:
@@ -756,7 +785,7 @@ class ModelAnalysisLogic:
*,
embedding_alignment: bool = False,
lora_mode: bool = False,
- ) -> tuple[str, str]:
+ ) -> tuple[str, str, str, str]:
paths = []
for name in (model_a, model_b):
path = folder_paths.get_full_path(model_type, name)
@@ -1037,7 +1066,9 @@ class ModelAnalysisLogic:
if embedding_alignment else "fixed tensor rows (index only)"
),
)
- return comparison, cwb_report
+ layerwise_metrics_csv = _csv_table(*_comparison_layerwise(records))
+ layerwise_cwb_csv = _csv_table(*_cwb_table_data((record.key, record.cwb) for record in cwb_records))
+ return comparison, cwb_report, layerwise_metrics_csv, layerwise_cwb_csv
finally:
for handler in handlers:
handler.__exit__(None, None, None)
@@ -1122,7 +1153,7 @@ class _ModelAnalysisNode(io.ComfyNode):
node_id=cls.NODE_ID,
display_name=cls.DISPLAY_NAME,
category="ModelUtils/Analysis",
- description="Compare two model files without merging them and return separate standard-metric and CWB-diagnostic reports.",
+ description="Compare two model files without merging them. Return Markdown reports and separate layerwise CSV text.",
inputs=_common_inputs(
cls.MODEL_TYPE,
alignment_control=cls.EMBEDDING_ALIGNMENT or cls.LORA_MODE,
@@ -1131,6 +1162,8 @@ class _ModelAnalysisNode(io.ComfyNode):
io.String.Output(display_name="comparison_report"),
io.String.Output(display_name="cwb_report"),
io.String.Output(display_name="documentation"),
+ io.String.Output(display_name="layerwise_metrics_csv", tooltip="CSV text for every comparable tensor; save as .csv with a text-saving node."),
+ io.String.Output(display_name="layerwise_cwb_csv", tooltip="CSV text for layerwise CWB diagnostics, with grouped metrics in separate columns."),
],
is_experimental=True,
)
@@ -1143,13 +1176,15 @@ class _ModelAnalysisNode(io.ComfyNode):
"Documentation mode active. No comparison performed.",
"Documentation mode active. No CWB analysis performed.",
documentation,
+ "",
+ "",
)
- comparison, cwb_report = ModelAnalysisLogic.execute(
+ comparison, cwb_report, metrics_csv, cwb_csv = ModelAnalysisLogic.execute(
kwargs["model_a"], kwargs["model_b"], cls.MODEL_TYPE, kwargs,
embedding_alignment=cls.EMBEDDING_ALIGNMENT,
lora_mode=cls.LORA_MODE,
)
- return io.NodeOutput(comparison, cwb_report, documentation)
+ return io.NodeOutput(comparison, cwb_report, documentation, metrics_csv, cwb_csv)
class CheckpointModelAnalysis(_ModelAnalysisNode):
diff --git a/tests/test_model_analysis.py b/tests/test_model_analysis.py
index a9d5326..6856844 100644
--- a/tests/test_model_analysis.py
+++ b/tests/test_model_analysis.py
@@ -1,4 +1,6 @@
import importlib
+import csv
+from io import StringIO
import math
import sys
import types
@@ -67,15 +69,17 @@ def test_five_node_schema_contract(analysis):
} & set(input_ids)
assert [item.display_name for item in schema.outputs] == [
"comparison_report", "cwb_report", "documentation",
+ "layerwise_metrics_csv", "layerwise_cwb_csv",
]
-def test_documentation_mode_has_three_separate_outputs(analysis):
+def test_documentation_mode_preserves_reports_and_returns_empty_csv(analysis):
result = analysis.DiffusionModelAnalysis.execute(execution_mode="DOCUMENTATION ONLY")
- assert len(result.result) == 3
+ assert len(result.result) == 5
assert "No comparison performed" in result[0]
assert "No CWB analysis performed" in result[1]
assert result[2].startswith("# Two-Model Similarity Analysis")
+ assert result[3] == result[4] == ""
def test_raw_metrics_have_independent_expected_values(analysis):
@@ -125,16 +129,17 @@ def test_zero_constant_and_nonfinite_metrics_are_explicit(analysis):
def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, analysis):
+ tensor_key = 'model.blocks.0.weight|"quoted",\nname'
path_a = tmp_path / "a.safetensors"
path_b = tmp_path / "b.safetensors"
save_file({
- "model.blocks.0.weight": torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
+ tensor_key: torch.tensor([[1.0, 2.0], [3.0, 4.0]]),
"only_a": torch.ones(1),
"shape": torch.ones(2),
"integer": torch.tensor([1, 2], dtype=torch.int64),
}, str(path_a), metadata={"source": "a"})
save_file({
- "model.blocks.0.weight": torch.tensor([[1.0, 0.0], [5.0, 4.0]]),
+ tensor_key: torch.tensor([[1.0, 0.0], [5.0, 4.0]]),
"only_b": torch.ones(1),
"shape": torch.ones(3),
"integer": torch.tensor([1, 3], dtype=torch.int64),
@@ -159,12 +164,29 @@ def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, a
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "mark_processed", track_processed)
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "async_stream", track_async_stream)
- comparison, cwb_report = analysis.ModelAnalysisLogic.execute(
- "a", "b", "diffusion_models", _params()
+ result = analysis.DiffusionModelAnalysis.execute(
+ model_a="a", model_b="b", execution_mode="ANALYZE", **_params()
)
- assert "Comparable floating tensors: 1" in comparison
- assert "mae: 1" in comparison
- assert "mse: 2" in comparison
+ comparison, cwb_report, documentation, metrics_csv, cwb_csv = result.result
+ assert documentation.startswith("# Two-Model Similarity Analysis")
+ assert comparison.startswith("# MODEL COMPARISON SUMMARY\n\n")
+ assert "| Comparable floating tensors | 1 |" in comparison
+ assert "| mae | 1 |" in comparison
+ assert "| mse | 2 |" in comparison
+ assert '| model.blocks.0.weight\\|"quoted",
name |' in comparison
+ assert "| --- | --- |" in comparison
+ metrics_rows = list(csv.DictReader(StringIO(metrics_csv)))
+ cwb_rows = list(csv.DictReader(StringIO(cwb_csv)))
+ assert len(metrics_rows) == len(cwb_rows) == 1
+ assert metrics_rows[0]["key"] == cwb_rows[0]["key"] == tensor_key
+ assert float(metrics_rows[0]["MAE"]) == 1
+ assert float(metrics_rows[0]["RMSE"]) == pytest.approx(math.sqrt(2), rel=1e-5)
+ assert metrics_rows[0]["dtype_A"] == metrics_rows[0]["dtype_B"] == "torch.float32"
+ assert cwb_rows[0]["matched"] == "N/A"
+ assert float(cwb_rows[0]["pair_cos_min"]) <= float(cwb_rows[0]["pair_cos_mean"]) <= float(cwb_rows[0]["pair_cos_max"])
+ top_rows = comparison.split("## TOP SCALAR WEIGHT DIFFERENCES\n\n", 1)[1].split("\n\n", 1)[0].splitlines()[2:]
+ assert len(top_rows) == 3
+ assert [float(row.rsplit(" | ", 4)[2]) for row in top_rows] == [2, 2, 0]
assert "[A ONLY] (1)" in comparison
assert "[B ONLY] (1)" in comparison
assert "[SHAPE MISMATCH] (1)" in comparison
@@ -175,7 +197,7 @@ def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, a
assert "PRESET-IMPLIED CONTRIBUTIONS" not in cwb_report
assert "Average contribution" not in cwb_report
assert len(processed) == 4
- assert {key for _, key in processed} == {"model.blocks.0.weight", "integer"}
+ assert {key for _, key in processed} == {tensor_key, "integer"}
assert len(async_calls) == 4
assert all(len(call[1]) == 1 for call in async_calls)
assert all(call[2]["batch_size"] == 1 for call in async_calls)
@@ -214,15 +236,17 @@ def test_lora_cross_format_keys_compare_as_one_logical_pair(monkeypatch, tmp_pat
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "mark_processed", track_processed)
monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "async_stream", track_async_stream)
- comparison, cwb_report = analysis.ModelAnalysisLogic.execute(
+ comparison, cwb_report, metrics_csv, cwb_csv = analysis.ModelAnalysisLogic.execute(
"a", "b", "loras", _params(), lora_mode=True,
)
- assert "Comparable floating tensors: 2" in comparison
+ assert "| Comparable floating tensors | 2 |" in comparison
assert "[A ONLY] (0)" in comparison
assert "[B ONLY] (0)" in comparison
assert "foo.down |" in comparison
assert "foo.up |" in comparison
- assert "foo.[down+up] |" in cwb_report
+ assert "foo.\\[down+up\\] |" in cwb_report
+ assert [row["key"] for row in csv.DictReader(StringIO(metrics_csv))] == ["foo.down", "foo.up"]
+ assert [row["key"] for row in csv.DictReader(StringIO(cwb_csv))] == ["foo.[down+up]"]
assert len(processed) == 4
assert len({entry for entry in processed}) == 4
assert len(async_calls) == 2
@@ -254,21 +278,23 @@ def test_excluded_lora_blocks_do_not_enter_metrics(
monkeypatch.setattr(analysis, "prepare_for_large_operation", lambda *args, **kwargs: None)
monkeypatch.setattr(analysis, "cleanup_after_operation", lambda: None)
- comparison, cwb_report = analysis.ModelAnalysisLogic.execute(
+ comparison, cwb_report, metrics_csv, cwb_csv = analysis.ModelAnalysisLogic.execute(
"a", "b", "loras",
_params(exclude_patterns=patterns, glob_patterns=glob_patterns, include_mode=include_mode),
lora_mode=True,
)
- assert "Comparable floating tensors: 2" in comparison
- assert "Excluded A tensors: 2" in comparison
- assert "Excluded B tensors: 2" in comparison
+ assert "| Comparable floating tensors | 2 |" in comparison
+ assert "| Excluded A tensors | 2 |" in comparison
+ assert "| Excluded B tensors | 2 |" in comparison
layerwise = comparison.split("LAYERWISE TENSOR METRICS", 1)[1].split(
"TOP SCALAR WEIGHT DIFFERENCES", 1
)[0]
- assert "blocks_3_" in layerwise
- assert "blocks_4_" not in layerwise
- assert "blocks_4_" not in cwb_report
+ assert "blocks\\_3\\_" in layerwise
+ assert "blocks\\_4\\_" not in layerwise
+ assert "blocks\\_4\\_" not in cwb_report
+ assert all("blocks_3_" in row["key"] for row in csv.DictReader(StringIO(metrics_csv)))
+ assert all("blocks_3_" in row["key"] for row in csv.DictReader(StringIO(cwb_csv)))
@pytest.mark.parametrize("patterns, glob_patterns, expected_keys", [