diff --git a/docs/model_analysis.md b/docs/model_analysis.md index 116dbc2..c5a6552 100644 --- a/docs/model_analysis.md +++ b/docs/model_analysis.md @@ -13,10 +13,18 @@ in both modes. ## Outputs `comparison_report` contains conventional numerical differences and topology -findings. `cwb_report` separately contains diagnostics derived from the +findings. Both reports use Markdown headings and tables. `cwb_report` separately contains diagnostics derived from the similarity, alignment, and consensus stages of Consensus-Weighted Blending. `documentation` contains this reference. +Two CSV-text outputs follow the existing three outputs, preserving their socket +positions: `layerwise_metrics_csv` and `layerwise_cwb_csv`. Connect them to a +text-saving node and use a `.csv` filename; analysis itself writes no files. +The CSV rows follow the same key order and displayed numeric precision as the +layerwise Markdown tables. Dtypes A/B, cosine mean/min/max, and A/B affinities +have separate columns. Undefined values remain `N/A`; an empty analysis produces +headers without data rows. Documentation-only mode returns empty CSV strings. + The reports begin with a global summary, followed by inferred blockwise values and one entry for every comparable tensor. Block names are approximated from the tensor-key hierarchy through its first numeric path component; they are not diff --git a/nodes/model_analysis.py b/nodes/model_analysis.py index 02a7ef6..4e3f460 100644 --- a/nodes/model_analysis.py +++ b/nodes/model_analysis.py @@ -2,12 +2,14 @@ from __future__ import annotations +import csv import gc import heapq import math import re from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass +from io import StringIO import comfy.utils import folder_paths @@ -580,14 +582,61 @@ def _average_layer_metrics(layer_stats: dict[str, RawStats]) -> dict[str, float return result -def _metadata_lines(metadata_a: dict | None, metadata_b: dict | None) -> list[str]: - a = metadata_a or {} - b = metadata_b or {} - keys = sorted(set(a) | set(b)) - differences = [key for key in keys if a.get(key) != b.get(key)] - lines = [f"Metadata keys: A={len(a)}, B={len(b)}, differing={len(differences)}"] - lines.extend(f"- {key}" for key in differences) - return lines +def _markdown_cell(value) -> str: + text = str(value).replace("&", "&").replace("<", "<").replace(">", ">") + for character in ("\\", "|", "`", "*", "_", "[", "]"): + text = text.replace(character, "\\" + character) + return text.replace("\r\n", "\n").replace("\r", "\n").replace("\n", "
") + + +def _markdown_table(headers, rows) -> str: + lines = [ + "| " + " | ".join(_markdown_cell(value) for value in headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + lines.extend("| " + " | ".join(_markdown_cell(value) for value in row) + " |" for row in rows) + return "\n".join(lines) + + +def _csv_table(headers, rows) -> str: + output = StringIO(newline="") + writer = csv.writer(output) + writer.writerow(headers) + writer.writerows(rows) + return output.getvalue() + + +def _comparison_layerwise(records): + headers = [ + "key", "shape", "dtype_A", "dtype_B", "MAE", "RMSE", "max_abs", + "relative_L2", "cosine", "pearson", "exact", "sign", "norm_B/A", + ] + rows = [] + for record in sorted(records, key=lambda item: item.key): + metric = _metrics(record.raw) + rows.append([ + record.key, _shape_text(record.shape), record.dtype_a, record.dtype_b, + *(_fmt(metric[name]) for name in ( + "mae", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", + )), + ]) + return headers, rows + + +def _cwb_table_data(entries, key_label="key"): + headers = [ + key_label, "pair_cos_mean", "pair_cos_min", "pair_cos_max", + "mean_aff_A", "mean_aff_B", "median_aff_A", "median_aff_B", + "matched", "alignment", "gain", + ] + rows = [] + for key, stats in sorted(entries, key=lambda item: item[0]): + values = stats.values() + rows.append([key, *(_fmt(values[name]) for name in ( + "pair_mean", "pair_min", "pair_max", "mean_aff_a", "mean_aff_b", + "median_aff_a", "median_aff_b", "matched", "alignment", "alignment_gain", + ))]) + return headers, rows def _render_comparison( @@ -606,86 +655,83 @@ def _render_comparison( ) -> str: weighted = _metrics(global_raw) layer_average = _average_layer_metrics(layer_stats) - lines = [ - "MODEL COMPARISON SUMMARY", - f"Type: {model_type}", - f"Model A: {model_a}", - f"Model B: {model_b}", - f"Comparable floating tensors: {len(records)}", - f"Comparable parameters: {global_raw.total}", - f"A-only keys: {len(inventory['a_only'])}", - f"B-only keys: {len(inventory['b_only'])}", - f"Shape mismatches: {len(inventory['shape'])}", - f"Non-floating shared tensors: {len(inventory['nonfloat'])}", - f"Unsupported low-bit tensors: {len(inventory['low_bit'])}", - f"Excluded A tensors: {len(inventory['excluded_a'])}", - f"Excluded B tensors: {len(inventory['excluded_b'])}", - f"CUDA OOM CPU fallbacks: {len(fallback_keys)}", - "", - "PARAMETER-WEIGHTED GLOBAL METRICS", + sections = [ + "# MODEL COMPARISON SUMMARY", + _markdown_table(["Field", "Value"], [ + ("Type", model_type), ("Model A", model_a), ("Model B", model_b), + ("Comparable floating tensors", len(records)), + ("Comparable parameters", global_raw.total), + ("A-only keys", len(inventory["a_only"])), + ("B-only keys", len(inventory["b_only"])), + ("Shape mismatches", len(inventory["shape"])), + ("Non-floating shared tensors", len(inventory["nonfloat"])), + ("Unsupported low-bit tensors", len(inventory["low_bit"])), + ("Excluded A tensors", len(inventory["excluded_a"])), + ("Excluded B tensors", len(inventory["excluded_b"])), + ("CUDA OOM CPU fallbacks", len(fallback_keys)), + ]), + "## PARAMETER-WEIGHTED GLOBAL METRICS", + _markdown_table(["Metric", "Value"], [(name, _fmt(weighted[name])) for name in ( + "mae", "mse", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", "coverage", + )]), + "## EQUAL-LAYER AVERAGES", + _markdown_table(["Metric", "Value"], [(name, _fmt(value)) for name, value in layer_average.items()]), + "## MOST DIVERGENT TENSORS", ] - for name in ("mae", "mse", "rmse", "max", "relative_l2", "cosine", "pearson", "exact", "sign", "norm_ratio", "coverage"): - lines.append(f"{name}: {_fmt(weighted[name])}") - lines.extend(["", "EQUAL-LAYER AVERAGES"]) - for name, value in layer_average.items(): - lines.append(f"{name}: {_fmt(value)}") - metrics_by_record = [(record, _metrics(record.raw)) for record in records] - lines.extend(["", "MOST DIVERGENT TENSORS", "By relative L2:"]) - for record, metric in sorted(metrics_by_record, key=lambda item: item[1]["relative_l2"] or -1.0, reverse=True)[:10]: - lines.append(f"- {record.key}: {_fmt(metric['relative_l2'])}") - lines.append("By MAE:") - for record, metric in sorted(metrics_by_record, key=lambda item: item[1]["mae"] or -1.0, reverse=True)[:10]: - lines.append(f"- {record.key}: {_fmt(metric['mae'])}") - lines.append("By lowest cosine:") - cosine_records = [(record, metric) for record, metric in metrics_by_record if metric["cosine"] is not None] - for record, metric in sorted(cosine_records, key=lambda item: item[1]["cosine"])[:10]: - lines.append(f"- {record.key}: {_fmt(metric['cosine'])}") - - lines.extend(["", "INFERRED BLOCKWISE METRICS", "block | params | MAE | RMSE | relative_L2 | cosine"]) + for title, metric_name, descending in ( + ("By relative L2", "relative_l2", True), + ("By MAE", "mae", True), + ("By lowest cosine", "cosine", False), + ): + ranked = sorted( + ((record, metric) for record, metric in metrics_by_record if metric[metric_name] is not None), + key=lambda item: item[1][metric_name], reverse=descending, + )[:10] + sections.extend([ + "### " + title, + _markdown_table(["Rank", "Tensor", metric_name], [ + (index, record.key, _fmt(metric[metric_name])) + for index, (record, metric) in enumerate(ranked, 1) + ]), + ]) + block_rows = [] for block in sorted(block_stats): raw = block_stats[block] metric = _metrics(raw) - lines.append( - f"{block} | {raw.total} | {_fmt(metric['mae'])} | {_fmt(metric['rmse'])} | " - f"{_fmt(metric['relative_l2'])} | {_fmt(metric['cosine'])}" - ) - - lines.extend(["", "LAYERWISE TENSOR METRICS", "key | shape | dtype A/B | MAE | RMSE | max_abs | relative_L2 | cosine | pearson | exact | sign | norm_B/A"]) - for record, metric in sorted(metrics_by_record, key=lambda item: item[0].key): - lines.append( - f"{record.key} | {_shape_text(record.shape)} | {record.dtype_a}/{record.dtype_b} | " - f"{_fmt(metric['mae'])} | {_fmt(metric['rmse'])} | {_fmt(metric['max'])} | " - f"{_fmt(metric['relative_l2'])} | {_fmt(metric['cosine'])} | {_fmt(metric['pearson'])} | " - f"{_fmt(metric['exact'])} | {_fmt(metric['sign'])} | {_fmt(metric['norm_ratio'])}" - ) - - lines.extend(["", "TOP SCALAR WEIGHT DIFFERENCES"]) - for value, key, flat_index, a_value, b_value, shape in sorted(top_weights, reverse=True): - lines.append( - f"- {key}{_coordinate(shape, flat_index)}: |delta|={_fmt(value)}, " - f"A={_fmt(a_value)}, B={_fmt(b_value)}" - ) - if not top_weights: - lines.append("- None") - - lines.extend(["", "TOPOLOGY AND DATA FINDINGS"]) + block_rows.append([block, raw.total, *(_fmt(metric[name]) for name in ("mae", "rmse", "relative_l2", "cosine"))]) + sections.extend([ + "## INFERRED BLOCKWISE METRICS", + _markdown_table(["block", "params", "MAE", "RMSE", "relative_L2", "cosine"], block_rows), + "## LAYERWISE TENSOR METRICS", + _markdown_table(*_comparison_layerwise(records)), + "## TOP SCALAR WEIGHT DIFFERENCES", + _markdown_table(["Rank", "Tensor", "Coordinate", "|delta|", "A", "B"], [ + (index, key, _coordinate(shape, flat_index), _fmt(value), _fmt(a_value), _fmt(b_value)) + for index, (value, key, flat_index, a_value, b_value, shape) in enumerate(sorted(top_weights, reverse=True), 1) + ]) if top_weights else "None.", + "## TOPOLOGY AND DATA FINDINGS", + ]) for label, name in ( ("A ONLY", "a_only"), ("B ONLY", "b_only"), ("SHAPE MISMATCH", "shape"), ("NON-FLOATING", "nonfloat"), ("LOW-BIT UNSUPPORTED", "low_bit"), ("EXCLUDED A", "excluded_a"), ("EXCLUDED B", "excluded_b"), ): - lines.append(f"[{label}] ({len(inventory[name])})") - lines.extend(f"- {value}" for value in inventory[name]) - if not inventory[name]: - lines.append("- None") - lines.append(f"[CUDA OOM -> CPU] ({len(fallback_keys)})") - lines.extend(f"- {key}" for key in fallback_keys) - if not fallback_keys: - lines.append("- None") - lines.extend(["", "METADATA"]) - lines.extend(_metadata_lines(metadata_a, metadata_b)) - return "\n".join(lines) + sections.extend([ + f"### [{label}] ({len(inventory[name])})", + _markdown_table(["Key / finding"], [(value,) for value in inventory[name]]) if inventory[name] else "None.", + ]) + sections.extend([ + f"### [CUDA OOM -> CPU] ({len(fallback_keys)})", + _markdown_table(["Key"], [(key,) for key in fallback_keys]) if fallback_keys else "None.", + "## METADATA", + ]) + a, b = metadata_a or {}, metadata_b or {} + differences = [key for key in sorted(set(a) | set(b)) if a.get(key) != b.get(key)] + sections.append(_markdown_table(["Metadata keys A", "Metadata keys B", "Differing keys"], [(len(a), len(b), len(differences))])) + if differences: + sections.append(_markdown_table(["Differing metadata key"], [(key,) for key in differences])) + return "\n\n".join(sections) + "\n" def _render_cwb( @@ -700,49 +746,32 @@ def _render_cwb( for record in records: global_cwb.add(record.cwb) values = global_cwb.values() - lines = [ - "CWB-DERIVED SIMILARITY SUMMARY", - f"Model A: {model_a}", - f"Model B: {model_b}", - f"Alignment scope: {alignment_scope}", - "Boundary: CWB similarity, alignment, and consensus diagnostics only.", - "Excluded: merge weighting, contribution calculation, vector reconstitution, norm rescaling, and output writing.", - "", - "GLOBAL CWB-DERIVED DIAGNOSTICS", - f"Pairwise row cosine mean/min/max: {_fmt(values['pair_mean'])} / {_fmt(values['pair_min'])} / {_fmt(values['pair_max'])}", - f"Mean-consensus affinity A/B: {_fmt(values['mean_aff_a'])} / {_fmt(values['mean_aff_b'])}", - f"Median-consensus affinity A/B: {_fmt(values['median_aff_a'])} / {_fmt(values['median_aff_b'])}", - f"Greedy alignment coverage: {_fmt(values['matched'])}", - f"Greedy matched cosine mean/min/max: {_fmt(values['alignment'])} / {_fmt(values['alignment_min'])} / {_fmt(values['alignment_max'])}", - f"Greedy improvement over index alignment: {_fmt(values['alignment_gain'])}", - "", - "INFERRED BLOCKWISE CWB DIAGNOSTICS", - "block | pair_cos mean/min/max | mean_aff_A/B | median_aff_A/B | matched | alignment | gain", + sections = [ + "# CWB-DERIVED SIMILARITY SUMMARY", + _markdown_table(["Field", "Value"], [ + ("Model A", model_a), ("Model B", model_b), ("Alignment scope", alignment_scope), + ("Boundary", "CWB similarity, alignment, and consensus diagnostics only."), + ("Excluded", "merge weighting, contribution calculation, vector reconstitution, norm rescaling, and output writing."), + ]), + "## GLOBAL CWB-DERIVED DIAGNOSTICS", + _markdown_table(["Metric", "Value"], [ + ("Pairwise row cosine mean/min/max", " / ".join(_fmt(values[name]) for name in ("pair_mean", "pair_min", "pair_max"))), + ("Mean-consensus affinity A/B", " / ".join(_fmt(values[name]) for name in ("mean_aff_a", "mean_aff_b"))), + ("Median-consensus affinity A/B", " / ".join(_fmt(values[name]) for name in ("median_aff_a", "median_aff_b"))), + ("Greedy alignment coverage", _fmt(values["matched"])), + ("Greedy matched cosine mean/min/max", " / ".join(_fmt(values[name]) for name in ("alignment", "alignment_min", "alignment_max"))), + ("Greedy improvement over index alignment", _fmt(values["alignment_gain"])), + ]), + "## INFERRED BLOCKWISE CWB DIAGNOSTICS", + _markdown_table(*_cwb_table_data(block_cwb.items(), "block")), + "## LAYERWISE CWB DIAGNOSTICS", + _markdown_table(*_cwb_table_data((record.key, record.cwb) for record in records)), + "## CUDA OOM CPU FALLBACKS", + f"Count: {len(fallback_keys)}", ] - for block in sorted(block_cwb): - entry = block_cwb[block].values() - lines.append( - f"{block} | {_fmt(entry['pair_mean'])}/{_fmt(entry['pair_min'])}/{_fmt(entry['pair_max'])} | " - f"{_fmt(entry['mean_aff_a'])}/{_fmt(entry['mean_aff_b'])} | " - f"{_fmt(entry['median_aff_a'])}/{_fmt(entry['median_aff_b'])} | " - f"{_fmt(entry['matched'])} | {_fmt(entry['alignment'])} | {_fmt(entry['alignment_gain'])}" - ) - lines.extend([ - "", "LAYERWISE CWB DIAGNOSTICS", - "key | pair_cos mean/min/max | mean_aff_A/B | median_aff_A/B | matched | alignment | gain", - ]) - for record in sorted(records, key=lambda value: value.key): - entry = record.cwb.values() - lines.append( - f"{record.key} | {_fmt(entry['pair_mean'])}/{_fmt(entry['pair_min'])}/{_fmt(entry['pair_max'])} | " - f"{_fmt(entry['mean_aff_a'])}/{_fmt(entry['mean_aff_b'])} | " - f"{_fmt(entry['median_aff_a'])}/{_fmt(entry['median_aff_b'])} | " - f"{_fmt(entry['matched'])} | {_fmt(entry['alignment'])} | {_fmt(entry['alignment_gain'])}" - ) - lines.append("") - lines.append(f"CUDA OOM CPU fallbacks: {len(fallback_keys)}") - lines.extend(f"- {key}" for key in fallback_keys) - return "\n".join(lines) + if fallback_keys: + sections.append(_markdown_table(["Key"], [(key,) for key in fallback_keys])) + return "\n\n".join(sections) + "\n" class ModelAnalysisLogic: @@ -756,7 +785,7 @@ class ModelAnalysisLogic: *, embedding_alignment: bool = False, lora_mode: bool = False, - ) -> tuple[str, str]: + ) -> tuple[str, str, str, str]: paths = [] for name in (model_a, model_b): path = folder_paths.get_full_path(model_type, name) @@ -1037,7 +1066,9 @@ class ModelAnalysisLogic: if embedding_alignment else "fixed tensor rows (index only)" ), ) - return comparison, cwb_report + layerwise_metrics_csv = _csv_table(*_comparison_layerwise(records)) + layerwise_cwb_csv = _csv_table(*_cwb_table_data((record.key, record.cwb) for record in cwb_records)) + return comparison, cwb_report, layerwise_metrics_csv, layerwise_cwb_csv finally: for handler in handlers: handler.__exit__(None, None, None) @@ -1122,7 +1153,7 @@ class _ModelAnalysisNode(io.ComfyNode): node_id=cls.NODE_ID, display_name=cls.DISPLAY_NAME, category="ModelUtils/Analysis", - description="Compare two model files without merging them and return separate standard-metric and CWB-diagnostic reports.", + description="Compare two model files without merging them. Return Markdown reports and separate layerwise CSV text.", inputs=_common_inputs( cls.MODEL_TYPE, alignment_control=cls.EMBEDDING_ALIGNMENT or cls.LORA_MODE, @@ -1131,6 +1162,8 @@ class _ModelAnalysisNode(io.ComfyNode): io.String.Output(display_name="comparison_report"), io.String.Output(display_name="cwb_report"), io.String.Output(display_name="documentation"), + io.String.Output(display_name="layerwise_metrics_csv", tooltip="CSV text for every comparable tensor; save as .csv with a text-saving node."), + io.String.Output(display_name="layerwise_cwb_csv", tooltip="CSV text for layerwise CWB diagnostics, with grouped metrics in separate columns."), ], is_experimental=True, ) @@ -1143,13 +1176,15 @@ class _ModelAnalysisNode(io.ComfyNode): "Documentation mode active. No comparison performed.", "Documentation mode active. No CWB analysis performed.", documentation, + "", + "", ) - comparison, cwb_report = ModelAnalysisLogic.execute( + comparison, cwb_report, metrics_csv, cwb_csv = ModelAnalysisLogic.execute( kwargs["model_a"], kwargs["model_b"], cls.MODEL_TYPE, kwargs, embedding_alignment=cls.EMBEDDING_ALIGNMENT, lora_mode=cls.LORA_MODE, ) - return io.NodeOutput(comparison, cwb_report, documentation) + return io.NodeOutput(comparison, cwb_report, documentation, metrics_csv, cwb_csv) class CheckpointModelAnalysis(_ModelAnalysisNode): diff --git a/tests/test_model_analysis.py b/tests/test_model_analysis.py index a9d5326..6856844 100644 --- a/tests/test_model_analysis.py +++ b/tests/test_model_analysis.py @@ -1,4 +1,6 @@ import importlib +import csv +from io import StringIO import math import sys import types @@ -67,15 +69,17 @@ def test_five_node_schema_contract(analysis): } & set(input_ids) assert [item.display_name for item in schema.outputs] == [ "comparison_report", "cwb_report", "documentation", + "layerwise_metrics_csv", "layerwise_cwb_csv", ] -def test_documentation_mode_has_three_separate_outputs(analysis): +def test_documentation_mode_preserves_reports_and_returns_empty_csv(analysis): result = analysis.DiffusionModelAnalysis.execute(execution_mode="DOCUMENTATION ONLY") - assert len(result.result) == 3 + assert len(result.result) == 5 assert "No comparison performed" in result[0] assert "No CWB analysis performed" in result[1] assert result[2].startswith("# Two-Model Similarity Analysis") + assert result[3] == result[4] == "" def test_raw_metrics_have_independent_expected_values(analysis): @@ -125,16 +129,17 @@ def test_zero_constant_and_nonfinite_metrics_are_explicit(analysis): def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, analysis): + tensor_key = 'model.blocks.0.weight|"quoted",\nname' path_a = tmp_path / "a.safetensors" path_b = tmp_path / "b.safetensors" save_file({ - "model.blocks.0.weight": torch.tensor([[1.0, 2.0], [3.0, 4.0]]), + tensor_key: torch.tensor([[1.0, 2.0], [3.0, 4.0]]), "only_a": torch.ones(1), "shape": torch.ones(2), "integer": torch.tensor([1, 2], dtype=torch.int64), }, str(path_a), metadata={"source": "a"}) save_file({ - "model.blocks.0.weight": torch.tensor([[1.0, 0.0], [5.0, 4.0]]), + tensor_key: torch.tensor([[1.0, 0.0], [5.0, 4.0]]), "only_b": torch.ones(1), "shape": torch.ones(3), "integer": torch.tensor([1, 3], dtype=torch.int64), @@ -159,12 +164,29 @@ def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, a monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "mark_processed", track_processed) monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "async_stream", track_async_stream) - comparison, cwb_report = analysis.ModelAnalysisLogic.execute( - "a", "b", "diffusion_models", _params() + result = analysis.DiffusionModelAnalysis.execute( + model_a="a", model_b="b", execution_mode="ANALYZE", **_params() ) - assert "Comparable floating tensors: 1" in comparison - assert "mae: 1" in comparison - assert "mse: 2" in comparison + comparison, cwb_report, documentation, metrics_csv, cwb_csv = result.result + assert documentation.startswith("# Two-Model Similarity Analysis") + assert comparison.startswith("# MODEL COMPARISON SUMMARY\n\n") + assert "| Comparable floating tensors | 1 |" in comparison + assert "| mae | 1 |" in comparison + assert "| mse | 2 |" in comparison + assert '| model.blocks.0.weight\\|"quoted",
name |' in comparison + assert "| --- | --- |" in comparison + metrics_rows = list(csv.DictReader(StringIO(metrics_csv))) + cwb_rows = list(csv.DictReader(StringIO(cwb_csv))) + assert len(metrics_rows) == len(cwb_rows) == 1 + assert metrics_rows[0]["key"] == cwb_rows[0]["key"] == tensor_key + assert float(metrics_rows[0]["MAE"]) == 1 + assert float(metrics_rows[0]["RMSE"]) == pytest.approx(math.sqrt(2), rel=1e-5) + assert metrics_rows[0]["dtype_A"] == metrics_rows[0]["dtype_B"] == "torch.float32" + assert cwb_rows[0]["matched"] == "N/A" + assert float(cwb_rows[0]["pair_cos_min"]) <= float(cwb_rows[0]["pair_cos_mean"]) <= float(cwb_rows[0]["pair_cos_max"]) + top_rows = comparison.split("## TOP SCALAR WEIGHT DIFFERENCES\n\n", 1)[1].split("\n\n", 1)[0].splitlines()[2:] + assert len(top_rows) == 3 + assert [float(row.rsplit(" | ", 4)[2]) for row in top_rows] == [2, 2, 0] assert "[A ONLY] (1)" in comparison assert "[B ONLY] (1)" in comparison assert "[SHAPE MISMATCH] (1)" in comparison @@ -175,7 +197,7 @@ def test_end_to_end_reports_and_topology_classification(monkeypatch, tmp_path, a assert "PRESET-IMPLIED CONTRIBUTIONS" not in cwb_report assert "Average contribution" not in cwb_report assert len(processed) == 4 - assert {key for _, key in processed} == {"model.blocks.0.weight", "integer"} + assert {key for _, key in processed} == {tensor_key, "integer"} assert len(async_calls) == 4 assert all(len(call[1]) == 1 for call in async_calls) assert all(call[2]["batch_size"] == 1 for call in async_calls) @@ -214,15 +236,17 @@ def test_lora_cross_format_keys_compare_as_one_logical_pair(monkeypatch, tmp_pat monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "mark_processed", track_processed) monkeypatch.setattr(analysis.MemoryEfficientSafeOpen, "async_stream", track_async_stream) - comparison, cwb_report = analysis.ModelAnalysisLogic.execute( + comparison, cwb_report, metrics_csv, cwb_csv = analysis.ModelAnalysisLogic.execute( "a", "b", "loras", _params(), lora_mode=True, ) - assert "Comparable floating tensors: 2" in comparison + assert "| Comparable floating tensors | 2 |" in comparison assert "[A ONLY] (0)" in comparison assert "[B ONLY] (0)" in comparison assert "foo.down |" in comparison assert "foo.up |" in comparison - assert "foo.[down+up] |" in cwb_report + assert "foo.\\[down+up\\] |" in cwb_report + assert [row["key"] for row in csv.DictReader(StringIO(metrics_csv))] == ["foo.down", "foo.up"] + assert [row["key"] for row in csv.DictReader(StringIO(cwb_csv))] == ["foo.[down+up]"] assert len(processed) == 4 assert len({entry for entry in processed}) == 4 assert len(async_calls) == 2 @@ -254,21 +278,23 @@ def test_excluded_lora_blocks_do_not_enter_metrics( monkeypatch.setattr(analysis, "prepare_for_large_operation", lambda *args, **kwargs: None) monkeypatch.setattr(analysis, "cleanup_after_operation", lambda: None) - comparison, cwb_report = analysis.ModelAnalysisLogic.execute( + comparison, cwb_report, metrics_csv, cwb_csv = analysis.ModelAnalysisLogic.execute( "a", "b", "loras", _params(exclude_patterns=patterns, glob_patterns=glob_patterns, include_mode=include_mode), lora_mode=True, ) - assert "Comparable floating tensors: 2" in comparison - assert "Excluded A tensors: 2" in comparison - assert "Excluded B tensors: 2" in comparison + assert "| Comparable floating tensors | 2 |" in comparison + assert "| Excluded A tensors | 2 |" in comparison + assert "| Excluded B tensors | 2 |" in comparison layerwise = comparison.split("LAYERWISE TENSOR METRICS", 1)[1].split( "TOP SCALAR WEIGHT DIFFERENCES", 1 )[0] - assert "blocks_3_" in layerwise - assert "blocks_4_" not in layerwise - assert "blocks_4_" not in cwb_report + assert "blocks\\_3\\_" in layerwise + assert "blocks\\_4\\_" not in layerwise + assert "blocks\\_4\\_" not in cwb_report + assert all("blocks_3_" in row["key"] for row in csv.DictReader(StringIO(metrics_csv))) + assert all("blocks_3_" in row["key"] for row in csv.DictReader(StringIO(cwb_csv))) @pytest.mark.parametrize("patterns, glob_patterns, expected_keys", [