From 4aff92199c4d71d1df775ae2bf4d86d57849f4ff Mon Sep 17 00:00:00 2001 From: Dariusz L Date: Sat, 15 Aug 2026 02:01:06 +0200 Subject: [PATCH] Add offline frontend and backend benchmarks --- benchmarks/README.md | 65 +++++++ benchmarks/backend.py | 212 +++++++++++++++++++++ benchmarks/browser.html | 394 ++++++++++++++++++++++++++++++++++++++++ benchmarks/frontend.mjs | 327 +++++++++++++++++++++++++++++++++ package.json | 2 + 5 files changed, 1000 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/backend.py create mode 100644 benchmarks/browser.html create mode 100644 benchmarks/frontend.mjs diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..7a2e222 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,65 @@ +# LayerForge offline benchmarks + +These benchmarks are deliberately separate from the functional test suite. They +measure the expensive data and pixel paths without starting ComfyUI or +downloading a model. + +## Frontend JavaScript + +This measures the real LayerForge mask pixel helpers, layer snapshot cloning, +and state signatures: + +```powershell +npm run benchmark -- --sizes 512,1024,2048 --layer-counts 1,4,8,16 --iterations 5 +``` + +Use `--format json` for machine-readable output. The Node benchmark does not +pretend that canvas stubs represent real rasterization; use the browser harness +for that part. + +## Backend Python + +Run it with the embedded Python used by the active Easy ComfyUI installation: + +```powershell +rtk proxy E:\AI\AI\ComfyUI\ComfyUI_Easy\All_Nodes\python_embeded\python.exe benchmarks\backend.py --sizes 512,1024,2048 --iterations 5 +``` + +The script measures tensor/data-URL conversion, per-sample and batch PNG +serialization, plus preprocessing/postprocessing around a fake RMBG model. It +never loads a checkpoint. Set `LAYERFORGE_COMFY_ROOT` if the active ComfyUI +root differs from the project default. + +## Real browser Canvas 2D and PNG export + +The browser harness measures representative compositing for 1, 4, 8, and 16 +layers, crop/blend/mask variants, PNG export, and the actual LayerForge distance +transform implementation. It is not a ComfyUI workflow test; it uses only a +local browser Canvas implementation. + +Start a static server with the repository explicitly selected as its document +root. This works even when the terminal is currently in another directory: + +```powershell +rtk proxy E:\AI\AI\ComfyUI\ComfyUI_Easy\All_Nodes\python_embeded\python.exe -m http.server 8765 --directory E:\AI\AI\ComfyUI\ComfyUI_Windows_portable\ComfyUI\custom_nodes\Comfyui-LayerForge +``` + +Open: + +```text +http://127.0.0.1:8765/benchmarks/browser.html +``` + +For the larger matrix, set the size field to `512,1024,2048`. Keep the same +browser, zoom, hardware acceleration setting, and iteration count when comparing +two revisions. Benchmark output is printed in the page and is not written to the +repository. + +## Interpreting results + +Compare median and p95 on the same machine. Warm-up samples are discarded. A +browser render median above roughly 16 ms is likely to affect 60 FPS interaction; +PNG export is a separate operation and should not be confused with the per-frame +render budget. Model inference is intentionally excluded from the fake matting +measurement and must be benchmarked separately with a specific checkpoint and +device. diff --git a/benchmarks/backend.py b/benchmarks/backend.py new file mode 100644 index 0000000..ca3139a --- /dev/null +++ b/benchmarks/backend.py @@ -0,0 +1,212 @@ +"""Offline LayerForge backend benchmark. + +This script measures tensor/image conversion and adapter overhead only. It does +not start ComfyUI, download models, or run a real matting checkpoint. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import time +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_COMFY_ROOT = Path(r"E:\AI\AI\ComfyUI\ComfyUI_Easy\All_Nodes\ComfyUI") + + +def _install_import_paths() -> Path: + comfy_root = Path(os.environ.get("LAYERFORGE_COMFY_ROOT", DEFAULT_COMFY_ROOT)) + sys.path.insert(0, str(REPOSITORY_ROOT)) + if comfy_root.exists(): + sys.path.insert(0, str(comfy_root)) + return comfy_root + + +def _parse_positive_list(value: str) -> list[int]: + result = [int(item.strip()) for item in value.split(",") if item.strip()] + if not result or any(item <= 0 for item in result): + raise argparse.ArgumentTypeError(f"Expected positive integers, got: {value}") + return result + + +def _percentile(sorted_values: list[float], fraction: float) -> float: + index = min(len(sorted_values) - 1, max(0, math.ceil(len(sorted_values) * fraction) - 1)) + return sorted_values[index] + + +def _measure(operation, iterations: int) -> dict[str, float]: + for _ in range(min(2, iterations)): + operation() + + samples = [] + for _ in range(iterations): + started_at = time.perf_counter() + operation() + samples.append((time.perf_counter() - started_at) * 1000) + + samples.sort() + return { + "median_ms": _percentile(samples, 0.5), + "p95_ms": _percentile(samples, 0.95), + "min_ms": samples[0], + "max_ms": samples[-1], + } + + +def _rounded(value): + if isinstance(value, dict): + return {key: _rounded(item) for key, item in value.items()} + if isinstance(value, float): + return round(value, 3) + return value + + +def _run_conversion_benchmarks(torch, image_utils, node_module, serialization, sizes, iterations): + from PIL import Image + + output = {} + for height, width in sizes: + tensor = torch.rand((1, 3, height, width), dtype=torch.float32) + encoded = image_utils.convert_tensor_to_base64(tensor) + sample = tensor[0].permute(1, 2, 0) + mask_tensor = torch.rand((height, width), dtype=torch.float32) + + def serialize_mask(): + mask_array = (mask_tensor.cpu().numpy() * 255).astype("uint8") + return serialization.pil_to_data_url(Image.fromarray(mask_array, "L")) + + output[f"{height}x{width}"] = { + "tensor_to_base64": _measure( + lambda: image_utils.convert_tensor_to_base64(tensor), iterations + ), + "base64_to_tensor": _measure( + lambda: image_utils.convert_base64_to_tensor(encoded), iterations + ), + "node_serialize_rgb_sample": _measure( + lambda: node_module.LayerForgeNode._serialize_rgb_tensor_sample(sample), iterations + ), + "mask_tensor_to_base64": _measure(serialize_mask, iterations), + "encoded_bytes": len(encoded.encode("ascii")), + } + + return output + + +def _run_batch_benchmarks(torch, node_module, batch_size: int, batch_counts, iterations): + output = {} + for batch_count in batch_counts: + batch = torch.rand((batch_count, batch_size, batch_size, 3), dtype=torch.float32) + + def serialize_batch(): + return [ + node_module.LayerForgeNode._serialize_rgb_tensor_sample(batch[index]) + for index in range(batch_count) + ] + + stats = _measure(serialize_batch, iterations) + output[str(batch_count)] = { + **stats, + "batch_size": batch_size, + "per_sample_median_ms": stats["median_ms"] / batch_count, + } + + return output + + +def _run_fake_matting_benchmarks(torch, rmbg_module, sizes, iterations): + class FakeModel: + def __call__(self, image): + return torch.zeros((image.shape[0], 1, 256, 256), device=image.device) + + adapter = rmbg_module.RMBG2Model(FakeModel(), torch.device("cpu")) + output = {} + for height, width in sizes: + image = torch.rand((1, height, width, 3), dtype=torch.float32) + output[f"{height}x{width}"] = _measure( + lambda: adapter.encode_image(image), iterations + ) + return output + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sizes", type=_parse_positive_list, default=[512, 1024, 2048]) + parser.add_argument("--iterations", type=int, default=5) + parser.add_argument("--batch-size", type=int, default=512) + parser.add_argument("--batch-counts", type=_parse_positive_list, default=[1, 4, 8, 16]) + parser.add_argument("--skip-matting", action="store_true") + args = parser.parse_args() + + if args.iterations < 1 or args.batch_size < 1: + parser.error("--iterations and --batch-size must be positive") + + comfy_root = _install_import_paths() + + try: + import torch + + from python import image_utils + from python import image_serialization as serialization + from python import node as node_module + except Exception as error: + print( + "Unable to import the LayerForge runtime. Set LAYERFORGE_COMFY_ROOT " + f"to the active ComfyUI root. Current root: {comfy_root}\n{error}", + file=sys.stderr, + ) + return 2 + + sizes = [(size, size) for size in args.sizes] + report = { + "benchmark": "layerforge-backend-offline", + "runtime": { + "python": sys.version.split()[0], + "torch": torch.__version__, + "torch_threads": torch.get_num_threads(), + "device": "cpu", + "comfy_root": str(comfy_root), + }, + "config": { + "sizes": args.sizes, + "iterations": args.iterations, + "batch_size": args.batch_size, + "batch_counts": args.batch_counts, + }, + "image_conversion": _run_conversion_benchmarks( + torch, image_utils, node_module, serialization, sizes, args.iterations + ), + "batch_serialization": _run_batch_benchmarks( + torch, node_module, args.batch_size, args.batch_counts, args.iterations + ), + "skipped": [], + } + + if args.skip_matting: + report["skipped"].append({ + "benchmark": "fake_matting_adapter", + "reason": "Disabled with --skip-matting", + }) + else: + try: + from python.matting.backends import rmbg as rmbg_module + + report["fake_matting_adapter"] = _run_fake_matting_benchmarks( + torch, rmbg_module, sizes, args.iterations + ) + except Exception as error: + report["skipped"].append({ + "benchmark": "fake_matting_adapter", + "reason": f"Adapter import failed: {error}", + }) + + print(json.dumps(_rounded(report), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/browser.html b/benchmarks/browser.html new file mode 100644 index 0000000..1992988 --- /dev/null +++ b/benchmarks/browser.html @@ -0,0 +1,394 @@ + + + + + + LayerForge Canvas Benchmark + + + +

LayerForge Canvas benchmark

+

+ This is a ComfyUI-free browser benchmark. It measures representative Canvas 2D + compositing and PNG export, plus the real LayerForge distance-transform helpers. + It is intentionally not a replacement for a ComfyUI workflow smoke test. +

+

+ + + + +

+
Ready.
+
+ 

+
+    
+
+
diff --git a/benchmarks/frontend.mjs b/benchmarks/frontend.mjs
new file mode 100644
index 0000000..cff9f0e
--- /dev/null
+++ b/benchmarks/frontend.mjs
@@ -0,0 +1,327 @@
+#!/usr/bin/env node
+
+import { performance } from 'node:perf_hooks';
+
+import {
+    applyLuminanceAsAlpha,
+    calculateDistanceTransform,
+    fillInverseAlphaMask,
+    rasterizeDistanceFieldMask,
+} from '../js/mask/mask_pixel_utils.js';
+import { HistoryStack } from '../js/canvas/canvas_history.js';
+import { cloneLayers, getStateSignature } from '../js/utils/common_utils.js';
+
+const DEFAULT_SIZES = [512, 1024, 2048];
+const DEFAULT_LAYER_COUNTS = [1, 4, 8, 16];
+const DEFAULT_ITERATIONS = 5;
+
+function parseList(value, fallback) {
+    if (value === undefined) return fallback;
+
+    const parsed = value
+        .split(',')
+        .map(item => Number.parseInt(item.trim(), 10))
+        .filter(Number.isFinite);
+
+    if (parsed.length === 0 || parsed.some(item => item <= 0)) {
+        throw new Error(`Expected a comma-separated list of positive integers, got: ${value}`);
+    }
+
+    return parsed;
+}
+
+function parseOptions(argv) {
+    const options = {
+        format: 'table',
+        sizes: DEFAULT_SIZES,
+        layerCounts: DEFAULT_LAYER_COUNTS,
+        iterations: DEFAULT_ITERATIONS,
+    };
+
+    for (let index = 0; index < argv.length; index += 1) {
+        const argument = argv[index];
+        if (argument === '--help' || argument === '-h') {
+            options.help = true;
+            continue;
+        }
+
+        const value = argv[index + 1];
+        if (value === undefined) {
+            throw new Error(`Missing value for ${argument}`);
+        }
+
+        if (argument === '--format') {
+            if (!['table', 'json'].includes(value)) {
+                throw new Error(`Unsupported format: ${value}`);
+            }
+            options.format = value;
+        } else if (argument === '--sizes') {
+            options.sizes = parseList(value, DEFAULT_SIZES);
+        } else if (argument === '--layer-counts') {
+            options.layerCounts = parseList(value, DEFAULT_LAYER_COUNTS);
+        } else if (argument === '--iterations') {
+            options.iterations = Number.parseInt(value, 10);
+            if (!Number.isFinite(options.iterations) || options.iterations < 1) {
+                throw new Error(`Expected --iterations to be a positive integer, got: ${value}`);
+            }
+        } else {
+            throw new Error(`Unknown option: ${argument}`);
+        }
+
+        index += 1;
+    }
+
+    return options;
+}
+
+function printHelp() {
+    console.log(`LayerForge offline frontend benchmark
+
+Usage:
+  npm run benchmark -- [options]
+
+Options:
+  --sizes 512,1024,2048       Pixel benchmark sizes
+  --layer-counts 1,4,8,16     History/model benchmark sizes
+  --iterations 5              Timed samples per operation
+  --format table|json         Output format (default: table)
+`);
+}
+
+function percentile(sortedValues, percentileValue) {
+    const index = Math.min(
+        sortedValues.length - 1,
+        Math.max(0, Math.ceil(sortedValues.length * percentileValue) - 1),
+    );
+    return sortedValues[index];
+}
+
+function measureSync(operation, iterations) {
+    const warmupCount = Math.min(2, iterations);
+    for (let index = 0; index < warmupCount; index += 1) {
+        operation();
+    }
+
+    const samples = [];
+    for (let index = 0; index < iterations; index += 1) {
+        const startedAt = performance.now();
+        operation();
+        samples.push(performance.now() - startedAt);
+    }
+
+    const sorted = samples.slice().sort((left, right) => left - right);
+    return {
+        medianMs: percentile(sorted, 0.5),
+        p95Ms: percentile(sorted, 0.95),
+        minMs: sorted[0],
+        maxMs: sorted[sorted.length - 1],
+    };
+}
+
+function createBinaryMask(size) {
+    const mask = new Uint8Array(size * size);
+    const center = size / 2;
+    const radius = size * 0.36;
+
+    for (let y = 0; y < size; y += 1) {
+        for (let x = 0; x < size; x += 1) {
+            const dx = x - center;
+            const dy = y - center;
+            mask[y * size + x] = dx * dx + dy * dy <= radius * radius ? 1 : 0;
+        }
+    }
+
+    return mask;
+}
+
+function createImageData(size) {
+    const data = new Uint8ClampedArray(size * size * 4);
+    for (let index = 0; index < data.length; index += 4) {
+        data[index] = index & 255;
+        data[index + 1] = (index >> 3) & 255;
+        data[index + 2] = (index >> 5) & 255;
+        data[index + 3] = 255;
+    }
+    return { data };
+}
+
+function benchmarkPixelOperations(size, iterations) {
+    const pixelCount = size * size;
+    const binaryMask = createBinaryMask(size);
+    const distanceMap = calculateDistanceTransform(binaryMask, size, size);
+    const rasterizedData = new Uint8ClampedArray(pixelCount * 4);
+    const imageData = createImageData(size);
+    const visibilityData = { data: new Uint8ClampedArray(pixelCount * 4) };
+    const maskData = { data: new Uint8ClampedArray(pixelCount * 4).fill(255) };
+
+    return {
+        distanceTransform: measureSync(
+            () => calculateDistanceTransform(binaryMask, size, size),
+            iterations,
+        ),
+        rasterizeDistanceField: measureSync(
+            () => rasterizeDistanceFieldMask(distanceMap, binaryMask, size * 0.1, rasterizedData),
+            iterations,
+        ),
+        luminanceToAlpha: measureSync(
+            () => applyLuminanceAsAlpha(imageData),
+            iterations,
+        ),
+        inverseAlphaMask: measureSync(
+            () => fillInverseAlphaMask(visibilityData, maskData),
+            iterations,
+        ),
+    };
+}
+
+function createLayerModel(count, size) {
+    return Array.from({ length: count }, (_, index) => ({
+        id: `benchmark-layer-${index}`,
+        x: (index % 4) * size * 0.04,
+        y: Math.floor(index / 4) * size * 0.04,
+        width: size * 0.72,
+        height: size * 0.72,
+        originalWidth: size,
+        originalHeight: size,
+        rotation: (index * 7) % 30,
+        zIndex: index,
+        blendMode: index % 3 === 0 ? 'normal' : 'multiply',
+        opacity: 0.7 + ((index % 3) * 0.1),
+        flipH: index % 5 === 0,
+        flipV: index % 7 === 0,
+        blendArea: index % 2 === 0 ? 25 : 0,
+        cropBounds: index % 3 === 0
+            ? { x: size * 0.1, y: size * 0.1, width: size * 0.8, height: size * 0.8 }
+            : undefined,
+        imageId: `benchmark-image-${index}`,
+        image: { src: 'data:image/png;base64,benchmark-placeholder' },
+    }));
+}
+
+function benchmarkHistory(count, size, iterations) {
+    const layers = createLayerModel(count, size);
+    const history = new HistoryStack({
+        clone: cloneLayers,
+        equals: (left, right) => getStateSignature(left) === getStateSignature(right),
+        historyLimit: 100,
+    });
+
+    return {
+        cloneLayers: measureSync(() => cloneLayers(layers), iterations),
+        stateSignature: measureSync(() => getStateSignature(layers), iterations),
+        pushSnapshot: measureSync(() => {
+            history.clear();
+            history.push(layers);
+        }, iterations),
+        estimatedSnapshotBytes: new TextEncoder().encode(JSON.stringify(cloneLayers(layers))).byteLength,
+    };
+}
+
+function roundStats(stats) {
+    return Object.fromEntries(
+        Object.entries(stats).map(([key, value]) => [
+            key,
+            typeof value === 'number' ? Number(value.toFixed(3)) : value,
+        ]),
+    );
+}
+
+function printTable(report) {
+    console.log('LayerForge offline frontend benchmark');
+    console.log(`Node ${report.runtime.node} on ${report.runtime.platform} ${report.runtime.arch}`);
+    console.log(`Iterations: ${report.config.iterations}`);
+    console.log('');
+    console.log('Pixel operations');
+    console.log('size  operation                 median   p95');
+
+    for (const [size, operations] of Object.entries(report.pixelOperations)) {
+        for (const [operation, stats] of Object.entries(operations)) {
+            console.log(
+                `${size.padStart(4)}  ${operation.padEnd(25)} `
+                + `${stats.medianMs.toFixed(3).padStart(7)} ms `
+                + `${stats.p95Ms.toFixed(3).padStart(7)} ms`,
+            );
+        }
+    }
+
+    console.log('');
+    console.log('Layer model/history operations');
+    console.log('layers  operation                 median   p95');
+    for (const [count, operations] of Object.entries(report.layerModel)) {
+        for (const operation of ['cloneLayers', 'stateSignature', 'pushSnapshot']) {
+            const stats = operations[operation];
+            console.log(
+                `${count.padStart(6)}  ${operation.padEnd(25)} `
+                + `${stats.medianMs.toFixed(3).padStart(7)} ms `
+                + `${stats.p95Ms.toFixed(3).padStart(7)} ms`,
+            );
+        }
+    }
+
+    console.log('');
+    console.log('Browser Canvas render/export: run benchmarks/browser.html in a browser.');
+}
+
+function main() {
+    const options = parseOptions(process.argv.slice(2));
+    if (options.help) {
+        printHelp();
+        return;
+    }
+
+    const report = {
+        benchmark: 'layerforge-frontend-offline',
+        runtime: {
+            node: process.version,
+            platform: process.platform,
+            arch: process.arch,
+        },
+        config: {
+            sizes: options.sizes,
+            layerCounts: options.layerCounts,
+            iterations: options.iterations,
+        },
+        pixelOperations: {},
+        layerModel: {},
+        skipped: [
+            {
+                benchmark: 'browser-canvas-render-export',
+                reason: 'Use benchmarks/browser.html with a real browser Canvas implementation.',
+            },
+        ],
+    };
+
+    for (const size of options.sizes) {
+        report.pixelOperations[size] = Object.fromEntries(
+            Object.entries(benchmarkPixelOperations(size, options.iterations))
+                .map(([name, stats]) => [name, roundStats(stats)]),
+        );
+    }
+
+    const historySize = options.sizes[0];
+    for (const count of options.layerCounts) {
+        report.layerModel[count] = {
+            ...benchmarkHistory(count, historySize, options.iterations),
+        };
+        report.layerModel[count] = Object.fromEntries(
+            Object.entries(report.layerModel[count]).map(([name, value]) => [
+                name,
+                typeof value === 'object' ? roundStats(value) : value,
+            ]),
+        );
+    }
+
+    if (options.format === 'json') {
+        console.log(JSON.stringify(report, null, 2));
+        return;
+    }
+
+    printTable(report);
+}
+
+try {
+    main();
+} catch (error) {
+    console.error(error instanceof Error ? error.message : error);
+    process.exitCode = 1;
+}
diff --git a/package.json b/package.json
index a1f7bcf..8552344 100644
--- a/package.json
+++ b/package.json
@@ -8,6 +8,8 @@
   "scripts": {
     "test": "node --test tests/*.mjs",
     "test:coverage": "node --experimental-test-coverage --test tests/*.mjs",
+    "benchmark": "node benchmarks/frontend.mjs",
+    "benchmark:frontend": "node benchmarks/frontend.mjs",
     "lint": "eslint js/ tests/ --max-warnings=0",
     "lint:fix": "eslint js/ tests/ --fix",
     "test:coverage:critical": "node --experimental-test-coverage --test-coverage-include=js/canvas/canvas_history.js --test-coverage-include=js/persistence/contracts.js --test-coverage-include=js/utils/web_socket_manager.js --test-coverage-lines=65 --test-coverage-functions=65 --test-coverage-branches=50 --test tests/test_canvas_history.mjs tests/test_persistence_contracts.mjs tests/test_persistence_resilience.mjs tests/test_async_lifecycle_resilience.mjs",