Merge branch 'feat/lazy-manifest' into int/wave3
This commit is contained in:
Jason Hoku
+2
-2
@@ -38,7 +38,7 @@ Read `ProjectStructure.md` in this directory for the full reference (file struct
|
||||
| `model_cache.py` | 935 | 3-tier cache: LoRA files → incremental states → patched models. Async preloading |
|
||||
| `manifest_utils.py` | 316 | Load/save/merge manifest. `save_manifest()` reloads from disk first to preserve concurrent user edits (favorites, votes, metrics) |
|
||||
| `metadata_packer.py` | 561 | PNG metadata embedding for CivitAI. `pack_metadata_into_image()` accepts `workflow_data` kwarg |
|
||||
| `run_api.py` | 144 | Pure helpers behind `POST /config_tester/run` and `GET /config_tester/run_status` |
|
||||
| `run_api.py` | 196 | Pure helpers behind `POST /config_tester/run`, `GET /config_tester/run_status` and `GET /config_tester/manifest` (`load_manifest_for_fetch`) |
|
||||
| `image_metrics.py`, `metrics_runner.py` | 668, 167 | Optional per-image metrics at save time; background job that backfills them for old sessions |
|
||||
| `workflow_sweep.py`, `workflow_sweep_runner.py`, `workflow_sweep_collect.py` | 1106, 1027, 281 | Workflow Sweep: pure expansion core, in-process queue feeder (never blocks the executor), `USCGSweepCollect` / `USCGSweepCollectVideo` output nodes |
|
||||
| `web/conf_builder/conf-builder-main.js` | 779 | Node registration. Default state and its migration blocks define ALL fields including upscaling/cooldown |
|
||||
@@ -144,7 +144,7 @@ Images saved as WebP at `output/benchmarks/{session}/images/img_{id}.webp`
|
||||
|
||||
- **Python files:** Restart ComfyUI
|
||||
- **`web/` JS files:** Refresh browser (Ctrl+F5)
|
||||
- **`resources/` files:** Must regenerate dashboard HTML — re-run sampler or call `/config_tester/get_session_html`
|
||||
- **`resources/` files:** Reload the dashboard (F5 on `/config_tester/dashboard`, or the node's RELOAD button). The page is a lazy shell: data comes from `GET /config_tester/manifest` (ETag/304, gzip); only code is inlined
|
||||
- **New state fields:** Add migration in `conf-builder-main.js` `onConfigure`
|
||||
|
||||
## Common Tasks
|
||||
|
||||
+7
-6
@@ -242,7 +242,8 @@ All routes registered via `PromptServer.instance.routes`. One row per route this
|
||||
| POST | `/config_tester/clear_comparisons` | `__init__.py` | `{session_name}` | Delete all votes of a session |
|
||||
| POST | `/config_tester/save_manifest` | `__init__.py` | `{session_name, manifest}` | Full merge save (keeps server-side `meta` and `comparisons`) |
|
||||
| POST | `/config_tester/get_session_html` | `__init__.py` | `{session_name, node_id}` | Generate the dashboard HTML on demand (empty manifest for an unknown session, not a 404) |
|
||||
| GET | `/config_tester/dashboard` | `__init__.py` | `?session=` (optional) | Dashboard SPA as a top-level page (standalone mode, `USCG_STANDALONE`, own websocket via `resources/logic_standalone.js`) |
|
||||
| GET | `/config_tester/dashboard` | `__init__.py` | `?session=` (optional) | Dashboard SPA as a top-level page (standalone mode, `USCG_STANDALONE`, own websocket via `resources/logic_standalone.js`). A lazy shell (no manifest inlined); gzip + content ETag (304 on `If-None-Match`) |
|
||||
| GET | `/config_tester/manifest` | `__init__.py` | `?session=` | A session's manifest as compact JSON for the SPA (lazy loading). Takes a session NAME, never a path. Weak `ETag` (mtime+size) + `If-None-Match` -> 304, gzip when accepted, `Cache-Control: no-cache`; 400 empty name, 403 outside benchmarks, 404 unknown, 503 + `Retry-After` for a torn read (`save_manifest` is not atomic; 3 retries first). Logic in `run_api.load_manifest_for_fetch` |
|
||||
| POST | `/config_tester/run` | `__init__.py` | `{configs_json or config_name, session_name?, overwrite_existing?, force? (default true), front?, client_id?}` | Headless run: builds the Generator+Dashboard prompt and enqueues it in-process like `/prompt`. Helpers in `run_api.py` |
|
||||
| GET | `/config_tester/run_status` | `__init__.py` | `?prompt_id=` | `pending`, `running`, `success` or `error` + session / dashboard_url / item_count |
|
||||
| POST | `/config_tester/export_favorites` | `__init__.py` | `{session_name, pack_metadata?, organize_by_prompt?, organize_by_lora?, export_prompt_txt?, copy_manifest? (default true), pack_workflow?, pack_nodes_workflow?, workflow_data?, nodes_workflows?}` | Export favorited images with options (metadata, workflow, organize) |
|
||||
@@ -329,10 +330,10 @@ All routes registered via `PromptServer.instance.routes`. One row per route this
|
||||
|
||||
|
||||
* **`html_generator.py`** (57 lines)
|
||||
* `get_html_template(title, manifest_data, node_id)` — reads all `/resources/` files, inlines into `template.html`
|
||||
* `get_html_template(title, manifest_data, node_id, standalone=False, inline_manifest=False)` — reads all `/resources/` files, inlines them into `template.html`. By default the page is a **lazy shell**: `fullManifest` is empty and flagged `__uscg_lazy__`, `manifest_data` is ignored, and the SPA fetches `GET /config_tester/manifest?session=<title>` itself (`inline_manifest=True` restores the old inlined data; used by tests)
|
||||
* **JS Load Order:** `logic_state.js` → `logic_utils.js` → `logic_ratings.js` → `logic_wfparams.js` → `logic_dragdrop.js` → `logic_ui.js` → `logic_upscale_modal.js` → `logic_virtual.js` → `logic_pipeline.js` → `logic_compare.js` → `logic_events.js` → `logic_init.js` → `logic_standalone.js` (all files share ONE inlined script: top-level names must be unique across them)
|
||||
* Replaces placeholders: `__CSS_CONTENT__`, `__JS_CONTENT__`, `__TITLE__`, `__NODE_ID__`
|
||||
* Zero external HTTP requests — everything inlined
|
||||
* Zero external HTTP requests for code (JS/CSS inlined); the manifest comes from `/config_tester/manifest`
|
||||
|
||||
---
|
||||
|
||||
@@ -438,7 +439,7 @@ The distribution code (job queue, worker thread, `/uscg-distribution/*` routes,
|
||||
|
||||
### **4. The Frontend: Dashboard SPA (`resources/`)**
|
||||
|
||||
The Dashboard is a standalone single-page app rendered inside an iframe. All files are inlined by `html_generator.py`.
|
||||
The Dashboard is a standalone single-page app rendered inside an iframe. All code files are inlined by `html_generator.py`; the manifest data is fetched lazily from `GET /config_tester/manifest` (boot in `logic_pipeline.js` `uscgBootLoad`, `fetchManifest`/`loadSession` in `logic_utils.js`, `reload_manifest` + `applyManifestReplace` in `logic_events.js`).
|
||||
|
||||
* **`template.html`** (662 lines)
|
||||
* HTML skeleton with settings panel (cog icon), export options, analytics
|
||||
@@ -825,7 +826,7 @@ __init__.py
|
||||
|
||||
3. **onNodeCreated Must Be Synchronous** — In `conf-builder-main.js` line 51, `onNodeCreated` MUST remain synchronous. Async initialization runs inside a fire-and-forget `(async () => { ... })()` IIFE. Making `onNodeCreated` itself async breaks widget registration.
|
||||
|
||||
4. **Resources Are Inlined** — `html_generator.py` reads all `/resources/` files and bakes them into one HTML string. Zero external HTTP requests. Changes to dashboard JS/CSS require re-generating the HTML (re-run the sampler node or call `/config_tester/get_session_html`).
|
||||
4. **Resources Are Inlined, Data Is Not** — `html_generator.py` reads all `/resources/` files and bakes them into one HTML string (~500 KB; the same string rides ComfyUI's `executed` websocket message and history, so it must not grow with the session). The manifest is NOT in it: the page is a lazy shell and `loadSession()` fetches `/config_tester/manifest` (compact, gzip, ETag). Changes to dashboard JS/CSS: F5 on `/config_tester/dashboard` or the node's RELOAD button rebuild the page from `resources/` on every request; a node that is already showing a page keeps the old JS until RELOAD. The Generator's `executed` message carries a shell, so `web/dashboard.js` turns it into a `reload_manifest` postMessage (conditional re-fetch) instead of parsing data out of the HTML.
|
||||
|
||||
5. **Manifest Merge on Save** — `manifest_utils.py` `save_manifest()` calls `merge_manifest_user_changes()` which reloads from disk before saving. This preserves user favorites/rejected/notes that changed concurrently while generation was running. Also, `flush_batch_with_vae()` (image_generation.py line 657) does its own disk sync.
|
||||
|
||||
@@ -904,7 +905,7 @@ __init__.py
|
||||
|
||||
- **Python backend files:** Restart ComfyUI
|
||||
- **`web/` JS files:** Refresh browser (Ctrl+F5) — served via `/extensions/` with cache-busting
|
||||
- **`resources/` JS/CSS/HTML files:** Dashboard HTML must be regenerated — re-run the sampler node or call `/config_tester/get_session_html`
|
||||
- **`resources/` JS/CSS/HTML files:** Reload the dashboard (F5 on `/config_tester/dashboard`, or the node's RELOAD button); a page that is already open keeps its old JS
|
||||
- **Config Builder state fields:** Also update migration checks in `conf-builder-main.js` `onConfigure`
|
||||
- **New config fields:** `state_to_configs_json()` in Python is the only converter (the JS preview calls `/configbuilder/preview`); then follow the checklist in gotcha 1
|
||||
|
||||
|
||||
@@ -988,6 +988,7 @@ http://<comfyui-host>:8188/config_tester/dashboard?session=<session_name>
|
||||
- It opens its own ComfyUI websocket, so new images and the ETA bar update live while a run is going. After your phone sleeps, it reloads the session when you come back, without duplicating cards. Reverse proxies that do not upgrade websockets (some Tailscale Serve / nginx setups) lose live updates; the **R** key and the resume reload still work.
|
||||
- Touch screens get 44px buttons, and the lightbox gets a bottom bar (Prev / Favorite / Reject / Revise / Next). The in-node view is unchanged apart from the extra header button.
|
||||
- **GENERATE NEW** in the lightbox queues through the run API below, into the session you are viewing. "Pack workflow" in export needs the ComfyUI tab and is disabled here.
|
||||
- **Lazy loading:** the page itself is only the dashboard code (~500 KB, ~115 KB gzipped); the image data comes from `GET /config_tester/manifest?session=<name>` (compact JSON, gzip, `ETag`). A 1000-image session is about 160 KB over the wire instead of 3.7 MB of page plus 3.5 MB of `manifest.json`, and a phone that wakes up and finds nothing new pays one 304. Both the page and the manifest answer `If-None-Match`, so a reload with nothing changed is nearly free. `--enable-compress-response-body` is not needed.
|
||||
- Like ComfyUI itself, the page has no authentication. Only expose it on a network you trust.
|
||||
|
||||
---
|
||||
@@ -1525,6 +1526,7 @@ Bot pattern: `POST /run`, post `dashboard_url` to the channel, poll `run_status`
|
||||
|---|---|---|
|
||||
| POST | `/configbuilder/preview` | Body `{state}` (the Builder state); returns `{configs_json}` from the same function the node uses (what the JSON Preview panel shows) |
|
||||
| GET | `/config_tester/dashboard` | The Dashboard as a standalone page, optional `?session=` |
|
||||
| GET | `/config_tester/manifest` | A session's manifest as compact JSON `?session=<name>` (gzip, ETag / 304); what the Dashboard fetches |
|
||||
| POST | `/config_tester/save_comparisons` | Append / remove Pairwise Review votes `{session_name, append, remove}` |
|
||||
| POST | `/config_tester/clear_comparisons` | Delete every vote of a session `{session_name}` |
|
||||
| POST | `/config_tester/compute_metrics` | Start the background metrics job for a session `{session_name, overwrite?}`; returns `{job_id, total_images}` |
|
||||
@@ -1712,7 +1714,7 @@ Tests live in `tests/` and must not contain network or process-spawning code (no
|
||||
|
||||
2. **One config-to-JSON implementation**: `UltimateConfigBuilder.state_to_configs_json` in `config_builder_node.py` is the single source of truth. The Builder preview POSTs the state to `/configbuilder/preview`, which runs the same function; there is no JS converter any more.
|
||||
|
||||
3. **Dashboard runs in an iframe** — The dashboard HTML is set via `iframe.srcdoc`. All JS/CSS is inlined (no external requests). Communication between the parent ComfyUI page and the iframe uses `postMessage()`. The `dashboard.js` file bridges server events to iframe messages.
|
||||
3. **Dashboard runs in an iframe** — The dashboard HTML is set via `iframe.srcdoc`. All JS/CSS is inlined (no external code requests); the manifest is fetched from `/config_tester/manifest` (the page is a lazy shell with no item data). Communication between the parent ComfyUI page and the iframe uses `postMessage()`. The `dashboard.js` file bridges server events to iframe messages.
|
||||
|
||||
4. **Dashboard auto-loads on node creation** — `web/dashboard.js` calls `forceLoadSession()` with a 500ms delay when a dashboard node is created. The `get_session_html` API endpoint returns a full HTML template with empty manifest for non-existent sessions (NOT a 404).
|
||||
|
||||
@@ -1782,7 +1784,7 @@ __init__.py
|
||||
|
||||
### Testing Changes
|
||||
|
||||
After modifying backend Python files, restart ComfyUI. After modifying `web/` JS files, refresh the browser (Ctrl+F5). After modifying `resources/` JS/CSS/HTML files, the dashboard HTML must be regenerated — either re-run the sampler node or call the `/config_tester/get_session_html` endpoint.
|
||||
After modifying backend Python files, restart ComfyUI. After modifying `web/` JS files, refresh the browser (Ctrl+F5). After modifying `resources/` JS/CSS/HTML files, reload the dashboard (F5 on `/config_tester/dashboard`, or the node's RELOAD button); the page is rebuilt from `resources/` on every request and fetches its data from `/config_tester/manifest`.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+3
-1
@@ -166,7 +166,9 @@ STARTED (11/4/26): the Pairwise Review scoreboard (per-axis win % + ratings) cov
|
||||
* **Milestone 3:** Hot-reload in dev — JS changes visible on browser refresh without re-running sampler
|
||||
Biggest architectural leverage: eliminates the #1 developer friction point.
|
||||
|
||||
STARTED (11/4/26): `/config_tester/dashboard` serves the SPA as a standalone page with its own websocket; the manifest is still inlined by html_generator (Milestones 1-3 open).
|
||||
STARTED (11/4/26): `/config_tester/dashboard` serves the SPA as a standalone page with its own websocket.
|
||||
DONE (11/4/26) Milestone 1: the page is a lazy shell (`html_generator`, `__uscg_lazy__`); the SPA fetches `GET /config_tester/manifest?session=` (compact JSON, gzip, ETag/304; the route is named `manifest`, not `get_manifest`). The node's `executed` message no longer carries the manifest (`web/dashboard.js` sends `reload_manifest`). Milestone 3 comes free for F5 / RELOAD (the page is rebuilt from `resources/` on every request); a node already showing a page keeps its old JS until RELOAD.
|
||||
Milestone 2 is blocked as written: ComfyUI's frontend imports every `web/**/*.js` as an extension module, so the `logic_*.js` files in `WEB_DIRECTORY` would run inside the ComfyUI page. Sketch of the cheap alternative: fixed path-less routes `GET /config_tester/dashboard_bundle_js?v=<hash>` / `..._css?v=<hash>` serving the concatenated files with ETag, `Cache-Control: public, max-age=31536000, immutable` and gzip, referenced from the shell with `<script src>` / `<link>` (the shell drops to ~70 KB and repeat loads to ~0 KB). Needs: rework `test_concatenated_script_parses_as_one_script`, the e2e fake origin serving the bundle, a script-order check.
|
||||
|
||||
#### **28. Onboarding Wizard & Tutorial**
|
||||
* **Milestone 1:** First-run wizard in Builder UI — walks through: create session → pick model → add LoRA → run → view dashboard (5 interactive steps)
|
||||
|
||||
+48
-8
@@ -1,5 +1,6 @@
|
||||
import re
|
||||
import asyncio
|
||||
import hashlib
|
||||
import server
|
||||
from aiohttp import web
|
||||
import json
|
||||
@@ -18,7 +19,7 @@ from .json_text_node import SmartJSONTextNode
|
||||
from .metadata_packer import pack_metadata_into_image
|
||||
from .directory_scanner import scan_directory_for_images
|
||||
from .run_api import (
|
||||
RunTracker, sanitize_session_name_strict, normalize_configs_payload, apply_run_overrides,
|
||||
RunTracker, sanitize_session_name_strict, load_manifest_for_fetch, normalize_configs_payload, apply_run_overrides,
|
||||
build_run_prompt, resolve_saved_config, saved_config_to_configs_obj, summarize_history, find_in_queue,
|
||||
)
|
||||
from .workflow_sweep_collect import USCGSweepCollect, USCGSweepCollectVideo
|
||||
@@ -631,20 +632,35 @@ async def get_session_html(request):
|
||||
# =============================================================================
|
||||
_run_tracker = RunTracker()
|
||||
|
||||
def _dashboard_response(request, html):
|
||||
"""200 text/html with a content ETag (304 on a match) and gzip when the client accepts it.
|
||||
|
||||
ComfyUI only compresses application/json and text/plain, never text/html, so the ~500 KB shell
|
||||
would go out raw. `enable_compression()` is aiohttp's own Accept-Encoding negotiation.
|
||||
"""
|
||||
etag = '"%s"' % hashlib.sha1(html.encode("utf-8")).hexdigest()[:20]
|
||||
headers = {"Cache-Control": "no-cache", "ETag": etag}
|
||||
if etag in [p.strip() for p in request.headers.get("If-None-Match", "").split(",")]:
|
||||
return web.Response(status=304, headers=headers)
|
||||
resp = web.Response(status=200, text=html, content_type="text/html", charset="utf-8", headers=headers)
|
||||
resp.enable_compression()
|
||||
return resp
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.get("/config_tester/dashboard")
|
||||
async def standalone_dashboard(request):
|
||||
"""
|
||||
Serve the dashboard SPA as a top-level page (phone / second-monitor friendly).
|
||||
Optional ?session=<name>; without it the SPA shows its session landing cards.
|
||||
The page is a SHELL (no manifest inlined): the SPA fetches /config_tester/manifest itself.
|
||||
"""
|
||||
try:
|
||||
session_name = sanitize_session_name_strict(request.query.get("session", ""))
|
||||
headers = {"Cache-Control": "no-store"}
|
||||
empty_manifest = {"items": [], "meta": {"model": "", "positive": "", "negative": ""}, "session_name": ""}
|
||||
|
||||
if not session_name:
|
||||
html = get_html_template("", empty_manifest, "standalone", standalone=True)
|
||||
return web.Response(status=200, text=html, content_type="text/html", charset="utf-8", headers=headers)
|
||||
return _dashboard_response(request, html)
|
||||
|
||||
base_dir = os.path.join(folder_paths.get_output_directory(), "benchmarks", session_name)
|
||||
if not _is_path_within(base_dir, _get_benchmarks_base()):
|
||||
@@ -655,17 +671,41 @@ async def standalone_dashboard(request):
|
||||
# Unknown session: landing page (picker); the title stays empty so the SPA does not
|
||||
# pretend a session is loaded.
|
||||
html = get_html_template("", empty_manifest, "standalone", standalone=True)
|
||||
return web.Response(status=200, text=html, content_type="text/html", charset="utf-8", headers=headers)
|
||||
return _dashboard_response(request, html)
|
||||
|
||||
with open(manifest_path, "r", encoding="utf-8") as f:
|
||||
manifest = json.load(f)
|
||||
html = get_html_template(session_name, manifest, "standalone", standalone=True)
|
||||
return web.Response(status=200, text=html, content_type="text/html", charset="utf-8", headers=headers)
|
||||
html = get_html_template(session_name, empty_manifest, "standalone", standalone=True)
|
||||
return _dashboard_response(request, html)
|
||||
except Exception as e:
|
||||
print(f"[GridTester] ❌ /dashboard failed: {e}")
|
||||
return web.Response(status=500, text=str(e))
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.get("/config_tester/manifest")
|
||||
async def get_manifest(request):
|
||||
"""
|
||||
A session's manifest.json as compact JSON for the dashboard SPA (lazy loading).
|
||||
?session=<name>. ETag + If-None-Match (304), gzip, retries a torn read (save_manifest is not
|
||||
atomic). The route takes a session NAME, never a path.
|
||||
"""
|
||||
try:
|
||||
raw = request.query.get("session", "")
|
||||
inm = request.headers.get("If-None-Match")
|
||||
for attempt in range(4):
|
||||
status, body, headers = load_manifest_for_fetch(_get_benchmarks_base(), raw, inm)
|
||||
if status != 503 or attempt == 3:
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
if body is None:
|
||||
return web.Response(status=status, headers=headers)
|
||||
resp = web.Response(status=status, body=body, content_type="application/json", charset="utf-8", headers=headers)
|
||||
if status == 200:
|
||||
resp.enable_compression()
|
||||
return resp
|
||||
except Exception as e:
|
||||
print(f"[GridTester] ❌ /manifest failed: {e}")
|
||||
return web.Response(status=500, text=str(e))
|
||||
|
||||
|
||||
@server.PromptServer.instance.routes.post("/config_tester/run")
|
||||
async def run_headless(request):
|
||||
"""
|
||||
|
||||
+10
-16
@@ -4,6 +4,12 @@ import json
|
||||
import folder_paths
|
||||
from .html_generator import get_html_template
|
||||
|
||||
# The template's <input id="session-input" class="session-name-input" type="text" value="__TITLE__">.
|
||||
# (The old pattern expected class="session-input" right after the id and never matched, so
|
||||
# update_session_name was always empty.)
|
||||
SESSION_INPUT_RE = re.compile(r'<input[^>]*\bid="session-input"[^>]*?\bvalue="([^"]*)"')
|
||||
|
||||
|
||||
class SamplerConfigDashboardViewer:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
@@ -26,7 +32,7 @@ class SamplerConfigDashboardViewer:
|
||||
# 1. If connected to Sampler (Fresh Generation)
|
||||
if dashboard_html:
|
||||
# --- MAGIC: EXTRACT SESSION NAME FROM HTML ---
|
||||
match = re.search(r'id="session-input" class="session-input" value="(.*?)"', dashboard_html)
|
||||
match = SESSION_INPUT_RE.search(dashboard_html)
|
||||
found_session = match.group(1) if match else None
|
||||
|
||||
# --- CRITICAL FIX: INJECT CORRECT DASHBOARD NODE ID ---
|
||||
@@ -50,20 +56,8 @@ class SamplerConfigDashboardViewer:
|
||||
session_name = re.sub(r'[^\w\-]', '', session_name)
|
||||
if not session_name: session_name = "default_session"
|
||||
|
||||
base_dir = os.path.join(folder_paths.get_output_directory(), "benchmarks", session_name)
|
||||
manifest_path = os.path.join(base_dir, "manifest.json")
|
||||
|
||||
if os.path.exists(manifest_path):
|
||||
try:
|
||||
with open(manifest_path, "r") as f:
|
||||
manifest = json.load(f)
|
||||
# Pass unique_id here, which get_html_template uses correctly
|
||||
html = get_html_template(session_name, manifest, unique_id)
|
||||
return {"ui": {"html": [html]}}
|
||||
except Exception as e:
|
||||
return {"ui": {"html": [f"Error loading session: {e}"]}}
|
||||
|
||||
# No session found — show full template with empty manifest so landing page can display
|
||||
# The page is a shell: the dashboard fetches the manifest itself (/config_tester/manifest),
|
||||
# so nothing is read here. An unknown session just shows the landing page.
|
||||
empty_manifest = {"items": [], "meta": {"model": "", "positive": "", "negative": ""}, "session_name": session_name}
|
||||
html = get_html_template(session_name, empty_manifest, unique_id)
|
||||
return {"ui": {"html": [html]}}
|
||||
return {"ui": {"html": [html]}}
|
||||
|
||||
+13
-2
@@ -1,9 +1,20 @@
|
||||
import os
|
||||
import json
|
||||
|
||||
def get_html_template(title, manifest_data, node_id, standalone=False):
|
||||
def get_html_template(title, manifest_data, node_id, standalone=False, inline_manifest=False):
|
||||
"""Build the dashboard page.
|
||||
|
||||
By default the page is a SHELL: no item data is inlined (an empty `fullManifest` flagged
|
||||
`__uscg_lazy__`), and the SPA fetches `/config_tester/manifest?session=<title>` itself. That
|
||||
keeps the HTML (and ComfyUI's `executed` websocket message / history entry, which carries the
|
||||
whole string) ~500 KB instead of growing with the session. `manifest_data` is only used when
|
||||
`inline_manifest=True` (old behaviour; offline export / tests).
|
||||
"""
|
||||
# 1. Normalize Data
|
||||
if isinstance(manifest_data, list):
|
||||
if not inline_manifest:
|
||||
manifest_data = {"items": [], "meta": {"model": "", "positive": "", "negative": ""},
|
||||
"session_name": str(title), "__uscg_lazy__": True}
|
||||
elif isinstance(manifest_data, list):
|
||||
manifest_data = {"items": manifest_data, "meta": {"model": "", "positive": "", "negative": ""}}
|
||||
|
||||
# Indent JSON for readability
|
||||
|
||||
@@ -697,6 +697,10 @@ function cmpKeyHandler(e) {
|
||||
|
||||
function openCompareMode() {
|
||||
if (_cmp.open) return;
|
||||
if (window._uscgBootPending) { // lazy shell: the manifest is still being fetched
|
||||
cmpSetStatus('Loading session...');
|
||||
return;
|
||||
}
|
||||
if (!fullManifest || !(fullManifest.items || []).length) {
|
||||
cmpSetStatus('Load a session with images first', true);
|
||||
return;
|
||||
|
||||
+43
-14
@@ -17,6 +17,35 @@ document.addEventListener('keydown', (e) => {
|
||||
// Track last update to prevent duplicates
|
||||
let lastUpdateId = 0;
|
||||
|
||||
/**
|
||||
* Replace the whole manifest with `manifest` WITHOUT touching filters, viewport or the grid layout
|
||||
* (the non-destructive path used by update_data{manifest}, reload_manifest and the standalone resync).
|
||||
* A lazy shell (`__uscg_lazy__`: empty on purpose) is ignored so it can never blank the grid.
|
||||
*/
|
||||
function applyManifestReplace(manifest, newMeta) {
|
||||
if (!manifest || manifest.__uscg_lazy__) return;
|
||||
const knownIds = new Set((activeData || []).map(function (it) { return it.id; }));
|
||||
fullManifest = manifest;
|
||||
activeData = fullManifest.items || [];
|
||||
meta = newMeta || manifest.meta || {};
|
||||
if (typeof onComparisonsReplaced === 'function') onComparisonsReplaced();
|
||||
|
||||
// Recompute label global values with new data
|
||||
if (typeof computeLabelGlobalValues === 'function' && labelMode && labelMode.enabled) {
|
||||
computeLabelGlobalValues();
|
||||
}
|
||||
|
||||
refreshIndices();
|
||||
|
||||
// Items we have not seen before may carry filter values (seed, sampler, ...) the filter sets do not
|
||||
// know yet: register them exactly like the live path does, or the pipeline would hide those cards.
|
||||
const fresh = activeData.filter(function (it) { return !knownIds.has(it.id); });
|
||||
if (fresh.length && typeof updateFiltersForNewData === 'function') updateFiltersForNewData(fresh);
|
||||
|
||||
// Full reprocess needed
|
||||
updateDataPipeline();
|
||||
}
|
||||
|
||||
// WebSocket Listener - OPTIMIZED
|
||||
window.addEventListener('message', (event) => {
|
||||
// Handle real-time ETA/progress updates
|
||||
@@ -48,6 +77,18 @@ window.addEventListener('message', (event) => {
|
||||
return;
|
||||
}
|
||||
|
||||
// The Generator finished (or the node was refreshed): re-fetch the manifest. If-None-Match makes an
|
||||
// unchanged manifest cost one 304; a changed one replaces the data without resetting the view.
|
||||
if (event.data && event.data.type === 'reload_manifest') {
|
||||
const sessInput = document.getElementById('session-input');
|
||||
const sess = sessInput ? sessInput.value : '';
|
||||
if (!sess) return;
|
||||
fetchManifest(sess, { ifChanged: true }).then(function (res) {
|
||||
if (res.status === 200 && res.data) applyManifestReplace(res.data, res.data.meta);
|
||||
}).catch(function (e) { console.warn('[Events] reload_manifest failed:', e); });
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.data && event.data.type === 'update_data') {
|
||||
const payload = event.data.data;
|
||||
if (!payload) return;
|
||||
@@ -82,21 +123,9 @@ window.addEventListener('message', (event) => {
|
||||
|
||||
// 2. Handle FULL MANIFEST reload (rare)
|
||||
if (payload.manifest) {
|
||||
if (payload.manifest.__uscg_lazy__) return; // lazy shell: no data, keep what we have
|
||||
console.log('[Events] 📚 Full manifest reload');
|
||||
fullManifest = payload.manifest;
|
||||
activeData = fullManifest.items || [];
|
||||
meta = payload.meta || {};
|
||||
if (typeof onComparisonsReplaced === 'function') onComparisonsReplaced();
|
||||
|
||||
// Recompute label global values with new data
|
||||
if (typeof computeLabelGlobalValues === 'function' && labelMode && labelMode.enabled) {
|
||||
computeLabelGlobalValues();
|
||||
}
|
||||
|
||||
refreshIndices();
|
||||
|
||||
// Full reprocess needed
|
||||
updateDataPipeline();
|
||||
applyManifestReplace(payload.manifest, payload.meta);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -84,6 +84,9 @@ function init() {
|
||||
restoredPosition = restoreViewportPosition();
|
||||
}
|
||||
|
||||
// (a lazy-shell boot has no data yet: its loadSession auto-fits once the data is in, unless this is true)
|
||||
window._uscgViewportRestored = restoredPosition;
|
||||
|
||||
// Only auto-fit if we didn't restore a saved position
|
||||
if (!restoredPosition) {
|
||||
setTimeout(() => {
|
||||
|
||||
+21
-12
@@ -822,18 +822,27 @@ function updateFiltersForNewData(newItems) {
|
||||
// --- 5. INITIALIZATION ---
|
||||
// Ensure init fires after DOM is ready
|
||||
// Note: init() in logic_init.js handles the full initialization including
|
||||
// showSessionLandingIfEmpty() for sessions with no data. Only call loadSession()
|
||||
// here when we have embedded data to reload from disk.
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
if (fullManifest && fullManifest.items && fullManifest.items.length > 0) {
|
||||
console.log('[Init] DOMContentLoaded - calling loadSession');
|
||||
loadSession();
|
||||
// showSessionLandingIfEmpty() for sessions with no data. The page is a lazy SHELL
|
||||
// (fullManifest.__uscg_lazy__, no item data inlined): when it names a session we fetch the
|
||||
// manifest here, once. _uscgBootPending is set synchronously so init()/the landing page know
|
||||
// data is on the way. A shell with an empty title is the landing page: no fetch.
|
||||
// Legacy pages that still inline items keep the old "reload from disk" call.
|
||||
function uscgBootLoad() {
|
||||
const sessInput = document.getElementById('session-input');
|
||||
const lazy = fullManifest && fullManifest.__uscg_lazy__;
|
||||
if (lazy) {
|
||||
if (sessInput && sessInput.value.trim()) {
|
||||
console.log('[Init] lazy shell - fetching manifest');
|
||||
window._uscgBootPending = true;
|
||||
loadSession(null, { boot: true });
|
||||
}
|
||||
});
|
||||
} else {
|
||||
if (fullManifest && fullManifest.items && fullManifest.items.length > 0) {
|
||||
console.log('[Init] DOM already loaded - calling loadSession');
|
||||
} else if (fullManifest && fullManifest.items && fullManifest.items.length > 0) {
|
||||
console.log('[Init] inlined manifest - calling loadSession');
|
||||
loadSession();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', uscgBootLoad);
|
||||
} else {
|
||||
uscgBootLoad();
|
||||
}
|
||||
|
||||
@@ -166,6 +166,8 @@ if (USCG_STANDALONE) {
|
||||
var isProgress = (name === 'ultimate_grid.progress');
|
||||
if (!isUpdate && !isProgress) return;
|
||||
|
||||
if (window._uscgBootPending) return; // the first manifest fetch is still in flight
|
||||
|
||||
// Landing page: follow a newly started session (mirrors the node bridge's "no session loaded" case).
|
||||
if (!window._uscgLoadedSession) {
|
||||
if (name === 'ultimate_grid.update' && data.session_name) {
|
||||
@@ -201,9 +203,14 @@ if (USCG_STANDALONE) {
|
||||
resyncTimer = setTimeout(function () {
|
||||
resyncTimer = null;
|
||||
if (navigator.onLine === false) return;
|
||||
// a background resync must not close the settings panel / session picker the user has open
|
||||
// a background resync must not close the settings panel / session picker the user has open,
|
||||
// nor reset filters/viewport: conditional fetch (304 = nothing to do), then a non-destructive swap
|
||||
window._uscgResyncing = true;
|
||||
Promise.resolve(loadSession()).finally(function () { window._uscgResyncing = false; });
|
||||
var sess = window._uscgLoadedSession;
|
||||
Promise.resolve(fetchManifest(sess, { ifChanged: true })).then(function (res) {
|
||||
if (res.status === 200 && res.data && window._uscgLoadedSession === sess) applyManifestReplace(res.data, res.data.meta);
|
||||
}).catch(function () { /* offline / session gone: keep showing what we have */ })
|
||||
.finally(function () { window._uscgResyncing = false; });
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
|
||||
+65
-13
@@ -50,10 +50,33 @@ async function saveState() {
|
||||
} catch (e) { console.error("Save failed", e); }
|
||||
}
|
||||
|
||||
// Load session from disk
|
||||
async function loadSession() {
|
||||
/**
|
||||
* Fetch a session's manifest from the lazy route (GET /config_tester/manifest?session=).
|
||||
* Compact JSON, gzip, ETag. With {ifChanged:true} it sends If-None-Match for the ETag of the
|
||||
* last manifest fetched for THIS session, so an unchanged manifest costs one 304 (data = null).
|
||||
* cache:'no-store' plus a hand-set If-None-Match makes the browser hand the 304 to us instead of
|
||||
* quietly turning it into a 200. {track:false} leaves the stored ETag alone (merge fetches).
|
||||
* Returns {status, data, etag}; status is the HTTP status (200 / 304 / 404 / ...).
|
||||
*/
|
||||
async function fetchManifest(sess, opts) {
|
||||
opts = opts || {};
|
||||
const prev = window._uscgManifestEtag;
|
||||
const headers = {};
|
||||
if (opts.ifChanged && prev && prev.session === sess && prev.etag) headers['If-None-Match'] = prev.etag;
|
||||
const r = await fetch('/config_tester/manifest?session=' + encodeURIComponent(sess), { cache: 'no-store', headers: headers });
|
||||
if (r.status === 304) return { status: 304, data: null, etag: prev ? prev.etag : null };
|
||||
if (!r.ok) return { status: r.status, data: null, etag: null };
|
||||
const data = await r.json();
|
||||
const etag = r.headers.get('ETag');
|
||||
if (etag && opts.track !== false) window._uscgManifestEtag = { session: sess, etag: etag };
|
||||
return { status: 200, data: data, etag: etag };
|
||||
}
|
||||
|
||||
// Load session from disk (opts.boot: first load of a lazy shell; a missing session just shows the landing page)
|
||||
async function loadSession(prefetched, opts) {
|
||||
const boot = !!(opts && opts.boot);
|
||||
const sessInput = document.getElementById('session-input');
|
||||
if (!sessInput) return;
|
||||
if (!sessInput) { if (boot) window._uscgBootPending = false; return; }
|
||||
const sess = sessInput.value;
|
||||
|
||||
console.log(`[Load] 🔄 Loading session: ${sess}`);
|
||||
@@ -63,9 +86,20 @@ async function loadSession() {
|
||||
if (grid) grid.style.opacity = '0.5';
|
||||
|
||||
try {
|
||||
const r = await fetch(`/view?filename=manifest.json&type=output&subfolder=benchmarks/${sess}&t=${Date.now()}`);
|
||||
if (!r.ok) throw new Error("Session not found");
|
||||
const data = await r.json();
|
||||
let data = prefetched;
|
||||
if (!data) {
|
||||
const res = await fetchManifest(sess);
|
||||
if (res.status !== 200) throw new Error("Session not found");
|
||||
data = res.data;
|
||||
}
|
||||
|
||||
// Boot only: live items that arrived (update_data) while the first fetch was in flight sit in
|
||||
// the still-empty fullManifest; keep the ones the fetched copy does not have.
|
||||
if (boot && fullManifest && Array.isArray(fullManifest.items) && fullManifest.items.length) {
|
||||
const have = new Set((data.items || []).map(function (it) { return it.id; }));
|
||||
const early = fullManifest.items.filter(function (it) { return !have.has(it.id); });
|
||||
if (early.length) data.items = early.concat(data.items || []);
|
||||
}
|
||||
|
||||
console.log(`[Load] 📊 Loaded ${data.items?.length || 0} items`);
|
||||
|
||||
@@ -173,11 +207,26 @@ async function loadSession() {
|
||||
|
||||
console.log('[Load] ✅ Session loaded successfully');
|
||||
|
||||
// Boot: init() ran its auto-fit on an empty grid; do it now unless a saved viewport was restored
|
||||
if (boot && !window._uscgViewportRestored && typeof autoFitZoom === 'function') autoFitZoom();
|
||||
|
||||
} catch (e) {
|
||||
console.error('[Load] ❌ Load failed:', e);
|
||||
if (grid) grid.style.opacity = '1';
|
||||
alert("Load Error: " + e.message);
|
||||
if (boot) {
|
||||
// unknown/unreadable session on first load: the landing page, no alert
|
||||
window._uscgBootPending = false;
|
||||
if (typeof showSessionLandingIfEmpty === 'function') showSessionLandingIfEmpty();
|
||||
} else {
|
||||
alert("Load Error: " + e.message);
|
||||
}
|
||||
} finally {
|
||||
if (boot) {
|
||||
const wasPending = window._uscgBootPending;
|
||||
window._uscgBootPending = false;
|
||||
// success path (the failure path already refreshed the landing page): populate the session picker now
|
||||
if (wasPending && typeof fetchAndShowSessions === 'function') fetchAndShowSessions();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,9 +240,9 @@ async function mergeSession(sessionName) {
|
||||
console.log(`[Merge] 🔄 Merging session: ${sessionName}`);
|
||||
|
||||
try {
|
||||
const r = await fetch(`/view?filename=manifest.json&type=output&subfolder=benchmarks/${sessionName}&t=${Date.now()}`);
|
||||
if (!r.ok) throw new Error("Session not found: " + sessionName);
|
||||
const data = await r.json();
|
||||
const res = await fetchManifest(sessionName, { track: false });
|
||||
if (res.status !== 200) throw new Error("Session not found: " + sessionName);
|
||||
const data = res.data;
|
||||
const newItems = data.items || [];
|
||||
|
||||
if (newItems.length === 0) {
|
||||
@@ -354,12 +403,15 @@ async function fetchAndShowSessions() {
|
||||
|
||||
// Show session landing page if no data is loaded
|
||||
function showSessionLandingIfEmpty() {
|
||||
const hasData = fullManifest && fullManifest.items && fullManifest.items.length > 0;
|
||||
// a lazy shell is still fetching its manifest: do not flash the landing cards meanwhile
|
||||
const hasData = window._uscgBootPending || (fullManifest && fullManifest.items && fullManifest.items.length > 0);
|
||||
const landing = document.getElementById('session-landing');
|
||||
const viewport = document.getElementById('viewport');
|
||||
|
||||
// Always populate session picker dropdown and landing cards
|
||||
fetchAndShowSessions();
|
||||
// Always populate session picker dropdown and landing cards. While the first manifest fetch is in
|
||||
// flight the list waits (loadSession fetches it right after): /list_sessions scans every session
|
||||
// folder on the server's event loop and would hold up the manifest response behind it.
|
||||
if (!window._uscgBootPending) fetchAndShowSessions();
|
||||
|
||||
if (!hasData && landing && viewport) {
|
||||
landing.style.display = 'block';
|
||||
|
||||
+52
@@ -17,6 +17,58 @@ def sanitize_session_name_strict(name):
|
||||
return re.sub(r'[^\w\-]', '', str(name or ''))
|
||||
|
||||
|
||||
def _etag_matches(if_none_match, etag):
|
||||
"""RFC 9110 weak comparison of an If-None-Match header against our ETag ('*' matches anything)."""
|
||||
if not if_none_match:
|
||||
return False
|
||||
want = etag[2:] if etag.startswith("W/") else etag
|
||||
for part in str(if_none_match).split(","):
|
||||
part = part.strip()
|
||||
if part == "*":
|
||||
return True
|
||||
if part.startswith("W/"):
|
||||
part = part[2:]
|
||||
if part == want:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def load_manifest_for_fetch(benchmarks_base, raw_session, if_none_match=None):
|
||||
"""Backend of GET /config_tester/manifest?session=<name> (the dashboard SPA's data fetch).
|
||||
|
||||
Returns (status, body_bytes_or_None, headers). One attempt only: status 503 means a torn read
|
||||
(the file changed while we read it) or invalid JSON (save_manifest is not atomic), which the
|
||||
route retries a few times before answering. The body is the manifest re-serialised COMPACT
|
||||
(the on-disk file is indent=4, ~1.9x bigger). ETag = weak validator from mtime_ns + size.
|
||||
"""
|
||||
session = sanitize_session_name_strict(raw_session)
|
||||
if not session:
|
||||
return 400, json.dumps({"error": "missing or invalid session"}).encode(), {}
|
||||
base = os.path.realpath(benchmarks_base)
|
||||
path = os.path.join(base, session, "manifest.json")
|
||||
real = os.path.realpath(path)
|
||||
if not real.startswith(base + os.sep):
|
||||
return 403, json.dumps({"error": "forbidden"}).encode(), {}
|
||||
try:
|
||||
st1 = os.stat(real)
|
||||
except OSError:
|
||||
return 404, json.dumps({"error": "not found"}).encode(), {}
|
||||
etag = 'W/"%x-%x"' % (st1.st_mtime_ns, st1.st_size)
|
||||
headers = {"ETag": etag, "Cache-Control": "no-cache"}
|
||||
if _etag_matches(if_none_match, etag):
|
||||
return 304, None, headers
|
||||
try:
|
||||
with open(real, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
st2 = os.stat(real)
|
||||
except (OSError, ValueError):
|
||||
return 503, json.dumps({"error": "manifest is being written, retry"}).encode(), {"Retry-After": "1"}
|
||||
if (st2.st_mtime_ns, st2.st_size) != (st1.st_mtime_ns, st1.st_size):
|
||||
return 503, json.dumps({"error": "manifest changed while reading, retry"}).encode(), {"Retry-After": "1"}
|
||||
body = json.dumps(data, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
return 200, body, headers
|
||||
|
||||
|
||||
def normalize_configs_payload(configs_json):
|
||||
"""Accept a dict with a `configs` list, a bare list, or a JSON string of either.
|
||||
|
||||
|
||||
+38
-4
@@ -25,9 +25,9 @@ def load_real_html_generator():
|
||||
return mod
|
||||
|
||||
|
||||
def build_dashboard_page(manifest, node_id="7"):
|
||||
"""The exact HTML string the Dashboard node shows in its iframe srcdoc."""
|
||||
return load_real_html_generator().get_html_template(SESSION, manifest, node_id)
|
||||
def build_dashboard_page(manifest, node_id="7", title=SESSION):
|
||||
"""The exact HTML string the Dashboard node shows in its iframe srcdoc (a lazy shell: no items inlined)."""
|
||||
return load_real_html_generator().get_html_template(title, manifest, node_id)
|
||||
|
||||
|
||||
def make_item(i, model, sampler, scheduler, cfg, steps, lora):
|
||||
@@ -101,6 +101,34 @@ class Dashboard:
|
||||
self.sessions = [] # payload of /config_tester/list_sessions
|
||||
self.comparisons = [] # votes echoed back by /config_tester/save_comparisons
|
||||
self.pageerrors = []
|
||||
# GET /config_tester/manifest (the lazy-manifest route): ETag W/"v<manifest_version>",
|
||||
# 304 on a matching If-None-Match. Bump manifest_version after changing self.manifest.
|
||||
self.manifest_version = 1
|
||||
self.manifest_requests = [] # {"session", "if_none_match"} per request
|
||||
self.title = SESSION # session name baked into the shell ("" = landing page)
|
||||
self.view_manifest_hits = 0 # old-style /view?filename=manifest.json requests (must stay 0 on boot)
|
||||
self.known_sessions = None # None: every session name exists (like the old /view); else a set
|
||||
self.hold_manifest = False # True: park manifest requests until release_manifest()
|
||||
self.held = []
|
||||
|
||||
def release_manifest(self):
|
||||
self.hold_manifest = False
|
||||
held, self.held = self.held, []
|
||||
for route in held:
|
||||
self._serve_manifest(route, route.request.url.partition("?")[2], None)
|
||||
|
||||
def _serve_manifest(self, route, query, _unused):
|
||||
from urllib.parse import parse_qs
|
||||
sess = (parse_qs(query).get("session") or [""])[0]
|
||||
inm = route.request.headers.get("if-none-match")
|
||||
self.manifest_requests.append({"session": sess, "if_none_match": inm})
|
||||
if self.known_sessions is not None and sess not in self.known_sessions:
|
||||
return route.fulfill(status=404, content_type="application/json", body='{"error": "not found"}')
|
||||
etag = f'W/"v{self.manifest_version}"'
|
||||
if inm and etag in [p.strip() for p in inm.split(",")]:
|
||||
return route.fulfill(status=304, headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||
return route.fulfill(status=200, content_type="application/json", body=json.dumps(self.manifest),
|
||||
headers={"ETag": etag, "Cache-Control": "no-cache"})
|
||||
|
||||
make_item = staticmethod(make_item)
|
||||
|
||||
@@ -126,11 +154,17 @@ class Dashboard:
|
||||
self.request_log.append((req.method, path, body))
|
||||
if path == "/":
|
||||
return route.fulfill(status=200, content_type="text/html; charset=utf-8",
|
||||
body=build_dashboard_page(self.manifest))
|
||||
body=build_dashboard_page(self.manifest, title=self.title))
|
||||
if path == "/config_tester/manifest":
|
||||
if self.hold_manifest:
|
||||
self.held.append(route)
|
||||
return None
|
||||
return self._serve_manifest(route, query, None)
|
||||
if path == "/view":
|
||||
params = dict(kv.partition("=")[::2] for kv in query.split("&") if kv)
|
||||
name = params.get("filename", "")
|
||||
if name == "manifest.json":
|
||||
self.view_manifest_hits += 1
|
||||
return route.fulfill(status=200, content_type="application/json",
|
||||
body=json.dumps(self.manifest))
|
||||
return route.fulfill(status=200, content_type="image/png", body=png_bytes(name))
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Lazy manifest loading: the page is a shell, the SPA fetches /config_tester/manifest itself.
|
||||
|
||||
Fake origin (see conftest.py: `Dashboard.handle` serves the route with ETag W/"v<n>" and 304).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
if os.environ.get("USCG_REQUIRE_E2E") == "1":
|
||||
import playwright.sync_api # noqa: F401
|
||||
else:
|
||||
pytest.importorskip("playwright.sync_api")
|
||||
|
||||
from playwright.sync_api import expect # noqa: E402
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
ORIGIN = "http://uscg.test"
|
||||
V1 = 'W/"v1"'
|
||||
|
||||
|
||||
def _open(page, kit, title="smoke", known=None, hold=False, goto=True):
|
||||
"""A Dashboard on the fake origin, not yet waited on. Returns (d, errors, dialogs)."""
|
||||
d = kit["Dashboard"](page, kit["make_manifest"]())
|
||||
d.title = title
|
||||
d.known_sessions = known
|
||||
d.hold_manifest = hold
|
||||
errors, dialogs = [], []
|
||||
page.on("pageerror", lambda e: errors.append(str(e)))
|
||||
page.on("dialog", lambda dlg: (dialogs.append(dlg.message), dlg.dismiss()))
|
||||
page.route("**/*", d.handle)
|
||||
page.set_viewport_size({"width": 1600, "height": 1200})
|
||||
if goto:
|
||||
page.goto(ORIGIN + "/")
|
||||
return d, errors, dialogs
|
||||
|
||||
|
||||
def _extra_item(d, item_id=5000, **overrides):
|
||||
"""A new item that introduces no new filter value (a clone of a non-euler item with a fresh id)."""
|
||||
src = next(it for it in d.manifest["items"] if it["sampler"] != "euler")
|
||||
item = dict(src, id=item_id, **overrides)
|
||||
return item
|
||||
|
||||
|
||||
def _wait_requests(page, d, n):
|
||||
for _ in range(80):
|
||||
if len(d.manifest_requests) >= n:
|
||||
return
|
||||
page.wait_for_timeout(100)
|
||||
raise AssertionError(f"expected {n} manifest requests, saw {d.manifest_requests}")
|
||||
|
||||
|
||||
# ---- (a) one manifest request on boot, and it is the new route -----------------------------------
|
||||
|
||||
def test_boot_does_one_manifest_request_and_no_view_manifest(dashboard):
|
||||
assert dashboard.manifest_requests == [{"session": "smoke", "if_none_match": None}]
|
||||
assert dashboard.view_manifest_hits == 0
|
||||
assert dashboard.ev("fullManifest.__uscg_lazy__") is None # replaced by the fetched manifest
|
||||
|
||||
|
||||
# ---- (b) landing shell (empty title) does no manifest request ---------------------------------------
|
||||
|
||||
def test_landing_shell_does_not_fetch(page, dashboard_kit):
|
||||
d, errors, dialogs = _open(page, dashboard_kit, title="")
|
||||
expect(page.locator("#session-landing")).to_be_visible()
|
||||
page.wait_for_timeout(400)
|
||||
assert d.manifest_requests == [] and d.view_manifest_hits == 0
|
||||
assert errors == [] and dialogs == []
|
||||
|
||||
|
||||
# ---- (c) unknown session: landing page, no alert, no errors ---------------------------------------
|
||||
|
||||
def test_unknown_session_shows_landing_without_alert(page, dashboard_kit):
|
||||
d, errors, dialogs = _open(page, dashboard_kit, title="ghost", known={"smoke"})
|
||||
expect(page.locator("#session-landing")).to_be_visible()
|
||||
assert [r["session"] for r in d.manifest_requests] == ["ghost"]
|
||||
assert page.evaluate("window._uscgBootPending") is False
|
||||
assert errors == [] and dialogs == []
|
||||
|
||||
|
||||
# ---- (d) slow fetch: no landing flash, 'c' is safe, cards arrive afterwards ----------------------------
|
||||
|
||||
def test_slow_manifest_no_landing_flash_and_compare_key_is_safe(page, dashboard_kit):
|
||||
d, errors, dialogs = _open(page, dashboard_kit, hold=True)
|
||||
page.wait_for_function("window._uscgBootPending === true")
|
||||
page.wait_for_timeout(600)
|
||||
assert len(d.held) == 1
|
||||
assert page.evaluate("getComputedStyle(document.getElementById('session-landing')).display") == "none"
|
||||
page.keyboard.press("c")
|
||||
assert page.evaluate("document.getElementById('cmp-status').textContent") == "Loading session..."
|
||||
assert not page.evaluate("!!window._compareOpen")
|
||||
d.release_manifest()
|
||||
page.wait_for_function("typeof processedData!=='undefined' && processedData.length===24")
|
||||
expect(page.locator("#grid .card:visible")).to_have_count(24)
|
||||
page.wait_for_function("window._uscgBootPending === false")
|
||||
assert errors == [] and dialogs == []
|
||||
|
||||
|
||||
def test_live_items_during_boot_fetch_are_kept(page, dashboard_kit):
|
||||
d, errors, _ = _open(page, dashboard_kit, hold=True)
|
||||
page.wait_for_function("window._uscgBootPending === true")
|
||||
extra = _extra_item(d, 7777)
|
||||
page.evaluate("(it) => window.postMessage({type: 'update_data', data: {new_items: [it]}}, '*')", extra)
|
||||
page.wait_for_timeout(200)
|
||||
d.release_manifest()
|
||||
page.wait_for_function("typeof processedData!=='undefined' && processedData.length===25")
|
||||
assert page.evaluate("activeData.some(x => x.id === 7777)")
|
||||
assert errors == []
|
||||
|
||||
|
||||
# ---- (e) standalone resync: conditional, silent on 304, filter-preserving on 200 ----------------------------
|
||||
|
||||
def test_resync_sends_if_none_match_and_keeps_view(dashboard):
|
||||
page = dashboard.page
|
||||
page.locator(".filter-popup-btn").click()
|
||||
page.locator("#btn-sampler-euler").click()
|
||||
expect(dashboard.visible_cards).to_have_count(16)
|
||||
dashboard.ev("toggleCogMenu()")
|
||||
cog_open = dashboard.ev("getComputedStyle(document.getElementById('cog-menu-dropdown')).display")
|
||||
assert cog_open != "none"
|
||||
assert len(dashboard.manifest_requests) == 1
|
||||
|
||||
page.evaluate("document.dispatchEvent(new Event('visibilitychange'))")
|
||||
_wait_requests(page, dashboard, 2)
|
||||
assert dashboard.manifest_requests[1]["if_none_match"] == V1
|
||||
page.wait_for_timeout(300)
|
||||
expect(dashboard.visible_cards).to_have_count(16) # 304: nothing re-rendered
|
||||
assert dashboard.ev("getComputedStyle(document.getElementById('cog-menu-dropdown')).display") == cog_open
|
||||
|
||||
# the manifest changes: the next resync is a 200, the new card shows and the euler filter is still on
|
||||
dashboard.manifest["items"].insert(0, _extra_item(dashboard))
|
||||
dashboard.manifest_version = 2
|
||||
page.evaluate("document.dispatchEvent(new Event('visibilitychange'))")
|
||||
page.wait_for_function("typeof processedData!=='undefined' && processedData.length===17")
|
||||
expect(dashboard.visible_cards).to_have_count(17)
|
||||
assert dashboard.ev("processedData.every(d => d.sampler !== 'euler')")
|
||||
assert dashboard.ev("getComputedStyle(document.getElementById('cog-menu-dropdown')).display") == cog_open
|
||||
assert dashboard.manifest_requests[-1]["if_none_match"] == V1
|
||||
|
||||
# an item with a NEW filter value (a seed nobody had) must not be hidden by the stale filter sets
|
||||
dashboard.manifest["items"].insert(0, _extra_item(dashboard, 5002, seed=987654))
|
||||
dashboard.manifest_version = 3
|
||||
page.evaluate("document.dispatchEvent(new Event('visibilitychange'))")
|
||||
page.wait_for_function("typeof processedData!=='undefined' && processedData.some(d => d.id === 5002)")
|
||||
|
||||
|
||||
# ---- (f) in the node iframe: reload_manifest from the parent, legacy update_data still works ----------------------
|
||||
|
||||
WRAP = """<!doctype html><body><iframe id="f" src="/" style="width:1400px;height:900px"></iframe></body>"""
|
||||
POST_RELOAD = "document.getElementById('f').contentWindow.postMessage({type: 'reload_manifest'}, '*')"
|
||||
|
||||
|
||||
def test_reload_manifest_message_and_legacy_update_data(page, dashboard_kit):
|
||||
d, errors, dialogs = _open(page, dashboard_kit, goto=False)
|
||||
page.route(ORIGIN + "/wrap", lambda route: route.fulfill(status=200, content_type="text/html", body=WRAP))
|
||||
page.goto(ORIGIN + "/wrap")
|
||||
fr = page.frame_locator("#f")
|
||||
fr.locator("#grid .card:visible").first.wait_for()
|
||||
inner = page.frames[1]
|
||||
inner.wait_for_function("typeof processedData!=='undefined' && processedData.length===24")
|
||||
assert inner.evaluate("USCG_STANDALONE") is False
|
||||
|
||||
fr.locator(".filter-popup-btn").click()
|
||||
fr.locator("#btn-sampler-euler").click()
|
||||
inner.wait_for_function("processedData.length===16")
|
||||
|
||||
# unchanged manifest: reload_manifest is a conditional request answered with 304 (nothing changes)
|
||||
page.evaluate(POST_RELOAD)
|
||||
_wait_requests(page, d, 2)
|
||||
assert d.manifest_requests[-1]["if_none_match"] == V1
|
||||
page.wait_for_timeout(200)
|
||||
assert inner.evaluate("processedData.length") == 16
|
||||
|
||||
# new item on disk: reload_manifest brings it in, filter kept
|
||||
d.manifest["items"].insert(0, _extra_item(d))
|
||||
d.manifest_version = 2
|
||||
page.evaluate(POST_RELOAD)
|
||||
inner.wait_for_function("processedData.length===17")
|
||||
assert inner.evaluate("processedData.every(x => x.sampler !== 'euler')")
|
||||
|
||||
# legacy full-manifest push still works
|
||||
full = json.loads(json.dumps(d.manifest))
|
||||
full["items"].insert(0, _extra_item(d, 5001))
|
||||
page.evaluate("(m) => document.getElementById('f').contentWindow.postMessage("
|
||||
"{type: 'update_data', data: {manifest: m, meta: m.meta}}, '*')", full)
|
||||
inner.wait_for_function("processedData.length===18")
|
||||
|
||||
# an (empty) lazy shell manifest must never blank the grid
|
||||
page.evaluate("document.getElementById('f').contentWindow.postMessage("
|
||||
"{type: 'update_data', data: {manifest: {items: [], meta: {}, __uscg_lazy__: true}}}, '*')")
|
||||
page.wait_for_timeout(300)
|
||||
assert inner.evaluate("processedData.length") == 18
|
||||
assert errors == [] and dialogs == []
|
||||
|
||||
|
||||
# ---- (g) auto-fit ran on real data: a card is inside the viewport ----------------------------------------------
|
||||
|
||||
def test_autofit_puts_cards_in_view_without_saved_position(dashboard):
|
||||
page = dashboard.page
|
||||
page.wait_for_timeout(300)
|
||||
inside = page.evaluate("""() => {
|
||||
const v = document.getElementById('viewport').getBoundingClientRect();
|
||||
return [...document.querySelectorAll('#grid .card')].filter(c => c.offsetParent !== null).some(c => {
|
||||
const r = c.getBoundingClientRect();
|
||||
return r.width > 0 && r.bottom > v.top && r.top < v.bottom && r.right > v.left && r.left < v.right;
|
||||
});
|
||||
}""")
|
||||
assert inside
|
||||
assert dashboard.ev("window._uscgViewportRestored") is False
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Pure backend of GET /config_tester/manifest (run_api.load_manifest_for_fetch)."""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
_PKG = os.path.basename(_ROOT)
|
||||
|
||||
|
||||
def _load(name, filename):
|
||||
spec = importlib.util.spec_from_file_location(f"{_PKG}.{name}_real_mf", os.path.join(_ROOT, filename))
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
mod.__package__ = _PKG
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
run_api = _load("run_api", "run_api.py")
|
||||
fetch = run_api.load_manifest_for_fetch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bench(tmp_path):
|
||||
base = tmp_path / "benchmarks"
|
||||
(base / "sess1").mkdir(parents=True)
|
||||
manifest = {"session_name": "sess1", "meta": {"model": "m"},
|
||||
"items": [{"id": 2, "note": "café ❤ </script>"}, {"id": 1, "note": ""}]}
|
||||
with open(base / "sess1" / "manifest.json", "w", encoding="utf-8") as f:
|
||||
json.dump(manifest, f, indent=4, ensure_ascii=False)
|
||||
(tmp_path / "secret").mkdir()
|
||||
(tmp_path / "secret" / "manifest.json").write_text('{"items": []}')
|
||||
return str(base), manifest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["", " ", None, "..", "../", "//", "<>"])
|
||||
def test_empty_after_sanitising_is_400(bench, raw):
|
||||
assert fetch(bench[0], raw)[0] == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["../secret", "..\\secret", "a/../../secret", "<script>", "sess1/../../secret"])
|
||||
def test_never_reads_outside_benchmarks(bench, raw):
|
||||
status, body, _ = fetch(bench[0], raw)
|
||||
assert status in (400, 403, 404)
|
||||
assert b"items" not in (body or b"")
|
||||
|
||||
|
||||
def test_junction_or_symlink_escape_is_403(bench, tmp_path):
|
||||
base, _ = bench
|
||||
link = os.path.join(base, "evil")
|
||||
try:
|
||||
os.symlink(str(tmp_path / "secret"), link, target_is_directory=True)
|
||||
except (OSError, NotImplementedError):
|
||||
import subprocess
|
||||
if os.name != "nt" or subprocess.run(["cmd", "/c", "mklink", "/J", link, str(tmp_path / "secret")],
|
||||
capture_output=True).returncode != 0:
|
||||
pytest.skip("cannot create a symlink or junction here")
|
||||
assert fetch(base, "evil")[0] == 403
|
||||
|
||||
|
||||
def test_missing_is_404_json(bench):
|
||||
status, body, _ = fetch(bench[0], "nope")
|
||||
assert status == 404 and json.loads(body) == {"error": "not found"}
|
||||
|
||||
|
||||
def test_ok_body_is_compact_and_equal(bench):
|
||||
base, manifest = bench
|
||||
status, body, headers = fetch(base, "sess1")
|
||||
assert status == 200
|
||||
assert json.loads(body.decode("utf-8")) == manifest
|
||||
assert b"\n" not in body and b": " not in body.replace("café".encode(), b"")
|
||||
assert "café ❤".encode("utf-8") in body # ensure_ascii=False keeps unicode
|
||||
assert headers["Cache-Control"] == "no-cache" and headers["ETag"].startswith('W/"')
|
||||
assert len(body) < os.path.getsize(os.path.join(base, "sess1", "manifest.json"))
|
||||
|
||||
|
||||
def test_etag_stable_then_changes_on_rewrite(bench):
|
||||
base, manifest = bench
|
||||
e1 = fetch(base, "sess1")[2]["ETag"]
|
||||
assert fetch(base, "sess1")[2]["ETag"] == e1
|
||||
path = os.path.join(base, "sess1", "manifest.json")
|
||||
manifest["items"].append({"id": 3})
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(manifest, f, indent=4)
|
||||
st = os.stat(path)
|
||||
os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns + 5_000_000_000))
|
||||
assert fetch(base, "sess1")[2]["ETag"] != e1
|
||||
|
||||
|
||||
def test_if_none_match_variants(bench):
|
||||
base, _ = bench
|
||||
etag = fetch(base, "sess1")[2]["ETag"]
|
||||
for inm in (etag, etag[2:], f'"zzz", {etag}', "*"):
|
||||
status, body, headers = fetch(base, "sess1", inm)
|
||||
assert status == 304 and body is None and headers["ETag"] == etag, inm
|
||||
status, body, _ = fetch(base, "sess1", 'W/"stale"')
|
||||
assert status == 200 and body
|
||||
|
||||
|
||||
def test_truncated_file_is_503_with_retry_after(bench):
|
||||
base, _ = bench
|
||||
path = os.path.join(base, "sess1", "manifest.json")
|
||||
raw = open(path, "rb").read()
|
||||
open(path, "wb").write(raw[: len(raw) // 2])
|
||||
status, _, headers = fetch(base, "sess1")
|
||||
assert status == 503 and headers["Retry-After"] == "1"
|
||||
|
||||
|
||||
def test_empty_file_is_503(bench):
|
||||
base, _ = bench
|
||||
open(os.path.join(base, "sess1", "manifest.json"), "wb").close()
|
||||
assert fetch(base, "sess1")[0] == 503
|
||||
@@ -20,8 +20,15 @@ NODE = shutil.which("node")
|
||||
|
||||
|
||||
def _page(**kw):
|
||||
"""The legacy inlined-manifest page (inline_manifest=True), for the escape/regex tests."""
|
||||
manifest = {"items": [{"id": 1, "note": EVIL}], "meta": {}}
|
||||
return html_generator.get_html_template("t", manifest, "standalone", **kw)
|
||||
return html_generator.get_html_template("t", manifest, "standalone", inline_manifest=True, **kw)
|
||||
|
||||
|
||||
def _shell(items=None, title="t", **kw):
|
||||
"""The default (lazy) page: manifest data passed in must NOT end up inlined."""
|
||||
manifest = {"items": items if items is not None else [{"id": 1, "note": EVIL}], "meta": {}}
|
||||
return html_generator.get_html_template(title, manifest, "standalone", **kw)
|
||||
|
||||
|
||||
def test_standalone_flag_and_no_placeholder_left():
|
||||
@@ -86,3 +93,92 @@ def test_dragdrop_helpers_inlined_before_logic_ui():
|
||||
assert 'draggable="true"' in html
|
||||
# node-only export guard stays harmless inline, helpers come before logic_ui's consumers
|
||||
assert html.index("function uscgImageRef(") < html.index("function sendModalToLoadImage(")
|
||||
|
||||
|
||||
# ---- lazy shell (default) -------------------------------------------------------------------
|
||||
def test_shell_inlines_no_item_data():
|
||||
page = _shell([{"id": 1, "note": "MARKER-NOTE-XYZ", "positive": "MARKER-PROMPT"}])
|
||||
assert "MARKER-NOTE-XYZ" not in page and "MARKER-PROMPT" not in page
|
||||
assert "__uscg_lazy__" in page
|
||||
assert "/*__JSON_START__*/" in page and "/*__JSON_END__*/" in page
|
||||
assert "__JSON_DATA__" not in page
|
||||
|
||||
|
||||
def test_shell_size_does_not_grow_with_the_session():
|
||||
big = [{"id": i, "note": "x" * 2500, "positive": "p" * 300} for i in range(1000)]
|
||||
assert len(_shell(big).encode("utf-8")) < 560 * 1024
|
||||
assert len(_shell(big)) == len(_shell([]))
|
||||
|
||||
|
||||
def test_shell_placeholders_all_replaced():
|
||||
page = html_generator.get_html_template("my-sess", {"items": []}, "42", standalone=True)
|
||||
assert 'const TARGET_NODE_ID = "42";' in page
|
||||
assert "const USCG_STANDALONE = true" in page
|
||||
assert 'id="session-input" class="session-name-input" type="text" value="my-sess"' in page
|
||||
for token in ("__TITLE__", "__NODE_ID__", "__STANDALONE_FLAG__", "__CSS_CONTENT__", "__JS_CONTENT__"):
|
||||
assert token not in page
|
||||
|
||||
|
||||
@pytest.mark.skipif(NODE is None, reason="node not installed")
|
||||
def test_shell_parses_with_dashboard_js_extraction_as_lazy():
|
||||
script = (
|
||||
"const html=require('fs').readFileSync(0,'utf8');"
|
||||
"const m=html.match(new RegExp(process.argv[1]));"
|
||||
"if(!m||!m[1]){console.error('no match');process.exit(2)}"
|
||||
"const o=JSON.parse(m[1]);process.stdout.write(JSON.stringify([o.__uscg_lazy__,o.items.length,o.session_name]));"
|
||||
)
|
||||
out = subprocess.run([NODE, "-e", script, _dashboard_js_pattern()], input=_shell(title="abc").encode("utf-8"),
|
||||
capture_output=True, timeout=30)
|
||||
assert out.returncode == 0, out.stderr.decode()
|
||||
assert out.stdout.decode() == '[true,0,"abc"]'
|
||||
|
||||
|
||||
@pytest.mark.skipif(NODE is None, reason="node not installed")
|
||||
def test_shell_script_parses_as_one_script():
|
||||
scripts = re.findall(r"<script>([\s\S]*?)</script>", _shell(standalone=True))
|
||||
js = max(scripts, key=len)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "page.js")
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(js)
|
||||
out = subprocess.run([NODE, "--check", path], capture_output=True, timeout=60)
|
||||
assert out.returncode == 0, out.stderr.decode()
|
||||
|
||||
|
||||
def test_inline_manifest_flag_keeps_the_old_behaviour():
|
||||
page = _page()
|
||||
assert "__uscg_lazy__" not in page.split("/*__JSON_START__*/")[1].split("/*__JSON_END__*/")[0]
|
||||
assert '"id": 1' in page
|
||||
|
||||
|
||||
# ---- dashboard_node: session name extraction from the generated page ---------------------------
|
||||
def _load_dashboard_node():
|
||||
import sys
|
||||
import types
|
||||
sys.modules.setdefault("folder_paths", types.SimpleNamespace(get_output_directory=lambda: "."))
|
||||
pkg = sys.modules.get(_PKG) or types.ModuleType(_PKG)
|
||||
pkg.__path__ = [_ROOT]
|
||||
sys.modules[_PKG] = pkg
|
||||
sys.modules[f"{_PKG}.html_generator"] = html_generator
|
||||
spec = importlib.util.spec_from_file_location(f"{_PKG}.dashboard_node_real", os.path.join(_ROOT, "dashboard_node.py"))
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
mod.__package__ = _PKG
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
@pytest.mark.parametrize("title", ["my_session", "Sexy-IL-v2"])
|
||||
def test_dashboard_node_extracts_session_name_and_sets_node_id(title):
|
||||
node = _load_dashboard_node()
|
||||
page = html_generator.get_html_template(title, {"items": []}, "gen-node")
|
||||
out = node.SamplerConfigDashboardViewer().view("ignored", "55", dashboard_html=page)["ui"]
|
||||
assert out["update_session_name"] == [title]
|
||||
assert 'const TARGET_NODE_ID = "55";' in out["html"][0]
|
||||
assert 'const TARGET_NODE_ID = "gen-node";' not in out["html"][0]
|
||||
|
||||
|
||||
def test_dashboard_node_landing_page_has_no_session_name():
|
||||
node = _load_dashboard_node()
|
||||
page = html_generator.get_html_template("", {"items": []}, "gen-node")
|
||||
out = node.SamplerConfigDashboardViewer().view("ignored", "55", dashboard_html=page)["ui"]
|
||||
assert out["update_session_name"] == []
|
||||
|
||||
@@ -430,6 +430,15 @@ app.registerExtension({
|
||||
try {
|
||||
const newManifest = JSON.parse(match[1]);
|
||||
|
||||
// Lazy shell (no item data inlined): never post its empty manifest as
|
||||
// data; tell the dashboard to re-fetch (it sends If-None-Match, so an
|
||||
// unchanged manifest costs one 304).
|
||||
if (newManifest.__uscg_lazy__ && node.iframe?.contentWindow) {
|
||||
node.iframe.contentWindow.postMessage({ type: 'reload_manifest' }, '*');
|
||||
console.log('[UltimateGrid] 🔄 Manifest re-fetch requested');
|
||||
return;
|
||||
}
|
||||
|
||||
// Send incremental update instead of full reload
|
||||
if (node.iframe?.contentWindow) {
|
||||
node.iframe.contentWindow.postMessage({
|
||||
|
||||
Reference in New Issue
Block a user