feat: module control — switch a module off per project, and a module that fails stays off across runs until repaired or turned back on (Settings > Modules)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.gitignore

@@ -36,6 +36,7 @@ app/playwright-report/
 /media/
 /shortcuts.json
 /markers.json
+/quarantine.json
 /shortcut_categories.json
 /shortcut_revolver.json
 

app/legacy/api.js

@@ -418,6 +418,7 @@
       document.body.appendChild(link); link.click(); link.remove();
       URL.revokeObjectURL(url);
     },
+    releaseModule: (id) => j("POST", API("/api/control/release"), { id }),
     deleteRefinementKey: (key) => j("DELETE", API(`/api/m/taste/keys/${encodeURIComponent(key)}`)),
     absoluteStoreInfo: () => unsupported("refinement keys are not built in v5 yet"),
     clearAbsoluteStore: () => unsupported("refinement keys are not built in v5 yet"),

app/legacy/engine_settings.js

@@ -37,7 +37,7 @@
 
   function categoriesWithContent() {
     const found = new Set();
-    Object.values(PS.modulesById()).forEach((m) => {
+    PS.activeModules().forEach((m) => {
       if (Object.keys(m.settings || {}).length) found.add(m.category || "");
     });
     return CATEGORY_ORDER.filter((c) => found.has(c));
@@ -58,7 +58,7 @@
     }
     if (!category || !cats.includes(category)) category = cats[0];
 
-    const modulesInCategory = Object.values(PS.modulesById())
+    const modulesInCategory = PS.activeModules()
       .filter((m) => (m.category || "") === category && Object.keys(m.settings || {}).length);
 
     const notes = PS.saveNotes();

app/legacy/index.html

@@ -154,6 +154,7 @@
   <script src="overlay_preview.js"></script>
   <script src="sampler_panel.js"></script>
   <script src="engine_settings.js"></script>
+  <script src="modules_control.js"></script>
   <script src="sampler_quickbar.js"></script>
   <script src="wheel.js"></script>
   <script src="wheel_actions_seed.js"></script>

app/legacy/modules_control.js

@@ -0,0 +1,70 @@
+// Settings ▸ Modules: switch a module off for THIS project (it vanishes from Engine settings and does
+// not run), and see the ones the server turned off after they failed, with a way to turn them back on.
+// Only modules that modify a run are listed: loaders, the sampler and the like are structure.
+(function () {
+  const { el, clear } = window.dom;
+  const PS = window.PipelineState;
+  const API = window.MovieEditorAPI;
+  const SF = window.SettingsField;
+
+  const CATEGORY_LABELS = {
+    continuity: "Continuity", guidance: "Guidance", conditioning: "Conditioning",
+    sampling: "Sampling", post: "Post", system: "System", "": "Other",
+  };
+  let _mounted = null;
+
+  function render() {
+    if (!_mounted) return;
+    const { content } = _mounted;
+    clear(content);
+    const pane = el("div", "models-pane eng-pane");
+    content.append(pane);
+    if (PS.loading()) { pane.append(SF.hintEl("Loading…")); return; }
+    if (PS.loadError()) { pane.append(SF.hintEl(`Could not load: ${PS.loadError()}`)); return; }
+    const control = PS.control();
+    const mine = Object.values(PS.modulesById()).filter((m) => control[m.id] && control[m.id].controllable);
+    if (!mine.length) { pane.append(SF.hintEl("No installed module can be switched off.")); return; }
+    pane.append(SF.hintEl("A module you turn off here is off for this project only, and disappears from Engine "
+      + "settings. One that failed while generating is turned off for every project until its code is "
+      + "repaired or you turn it back on."));
+    const byCat = {};
+    mine.forEach((m) => { (byCat[m.category || ""] = byCat[m.category || ""] || []).push(m); });
+    Object.keys(CATEGORY_LABELS).filter((c) => byCat[c]).forEach((c) => {
+      const g = SF.group(pane, CATEGORY_LABELS[c]);
+      byCat[c].forEach((m) => {
+        const held = control[m.id].quarantine;
+        if (held) {
+          const back = el("button", "btn ghost tiny", "Turn back on");
+          back.onclick = async () => {
+            try { await API.releaseModule(m.id); } catch (e) { alert(`Could not turn ${m.title || m.id} back on: ${e.message || e}`); }
+            await PS.refreshControl(); render();
+          };
+          g.append(SF.field(`${m.title || m.id} — failed`, back,
+            `Off since ${held.when || "a failure"}: ${held.reason || "an error"}`));
+          return;
+        }
+        const cb = el("input"); cb.type = "checkbox"; cb.checked = !PS.isOff(m.id);
+        cb.onchange = () => { PS.setOff(m.id, !cb.checked).then(render); };
+        g.append(SF.toggleField(m.title || m.id, cb, PS.isOff(m.id) ? "Off for this project" : undefined));
+      });
+    });
+  }
+
+  function mount(body) {
+    const content = el("div", "models-mount eng-mount");
+    body.append(content);
+    _mounted = { content };
+    render();
+    PS.ensureLoaded().then(() => PS.refreshControl()).then(render);
+    return () => { _mounted = null; };
+  }
+
+  window.SettingsWindow.register({
+    id: "modules", group: "Generation", order: 3, title: "Modules",
+    subtitle: "Switch a module off for this project; see and re-enable ones that failed.",
+    keywords: "modules enable disable off quarantine failed repair switch",
+    iconBg: "linear-gradient(180deg,#6fd08c,#2f9e5a)",
+    icon: '<svg viewBox="0 0 16 16" width="13" height="13" fill="none" stroke="#fff" stroke-width="1.5" stroke-linecap="round"><path d="M3 4h10M3 8h10M3 12h10"/><circle cx="6" cy="4" r="1.4" fill="#fff"/><circle cx="10.5" cy="8" r="1.4" fill="#fff"/><circle cx="5" cy="12" r="1.4" fill="#fff"/></svg>',
+    mount,
+  });
+})();

app/legacy/pipeline_state.js

@@ -58,6 +58,7 @@
     const manifest = await API.modules(traits ? traits.join(",") : undefined);
     modulesById = {};
     (manifest.modules || []).forEach((m) => { modulesById[m.id] = m; });
+    controlState = manifest.control || controlState;
     lastProbedFile = file;                              // only once it worked: a failed probe is tried again
   }
   let incomplete = [];
@@ -85,7 +86,7 @@
   // session already knows is installed, so an unrelated node's incidentally
   // JSON-shaped string can't inject a bogus entry that round-trips forever.
   function valuesAlreadyPlaced() {
-    const known = new Set(Object.keys(modulesById));
+    const known = new Set([...Object.keys(modulesById), "_off"]);    // _off: the modules this project turned off
     const merged = {};
     (slots || []).forEach((slot) => {
       Object.values(slot.inputs || {}).forEach((v) => {
@@ -231,7 +232,9 @@
     Object.keys(pendingValues).forEach((moduleId) => {
       const pend = pendingValues[moduleId];
       const real = already[moduleId] || {};
-      Object.keys(pend).forEach((name) => { if (real[name] === pend[name]) delete pend[name]; });
+      Object.keys(pend).forEach((name) => {
+        if (real[name] === pend[name] || (typeof pend[name] === "object" && JSON.stringify(real[name]) === JSON.stringify(pend[name]))) delete pend[name];
+      });
       if (!Object.keys(pend).length) delete pendingValues[moduleId];
     });
   }
@@ -261,6 +264,27 @@
     return merged;
   }
 
+  // Which modules this project turned off (a reserved entry of the same settings tree), and which
+  // the server has quarantined after a fault. Neither shows in Engine settings or the quick bar.
+  let controlState = {};
+  function isOff(id) {
+    const off = (currentValues()._off || {}).modules;
+    return Array.isArray(off) && off.includes(id);
+  }
+  function setOff(id, off) {
+    const cur = new Set(((currentValues()._off || {}).modules) || []);
+    if (off) cur.add(id); else cur.delete(id);
+    return setModuleValue("_off", "modules", [...cur]);
+  }
+  async function refreshControl() {
+    try { controlState = (await API.modules()).control || {}; } catch (_) { /* keeps the last answer */ }
+    return controlState;
+  }
+  function isQuarantined(id) { return !!(controlState[id] && controlState[id].quarantine); }
+  function activeModules() {
+    return Object.values(modulesById).filter((m) => !isOff(m.id) && !isQuarantined(m.id));
+  }
+
   // Patch ONE field and save the WHOLE tree. place() (core/graph.py) writes
   // values as a single opaque JSON blob with no merge on the server side --
   // sending only the module being edited would silently erase every other
@@ -449,6 +473,7 @@
     }).concat(extras ? saved.filter((s) => sound(s) && !have.has(s.id)) : []);
     // Replaces whatever pipeline was live: nothing queued for the old one may land on the new.
     const mine = ++epoch;
+    pendingValues = {};                                 // the new project's values win, not an edit made to the old one
     removed = new Set(gone);
     unwired = {};
     Object.entries(unwiredMap && typeof unwiredMap === "object" ? unwiredMap : {}).forEach(([id, list]) => {
@@ -482,6 +507,7 @@
   window.PipelineState = {
     ensureLoaded, save, edit, restore, snapshot, setGroup, adopt, subscribe: (fn) => { listeners.add(fn); return () => listeners.delete(fn); }, valuesAlreadyPlaced, currentValues, setModuleValue,
     modulesById: () => modulesById,
+    activeModules, isOff, setOff, refreshControl, control: () => controlState,
     removedIds: () => [...removed],
     unwiredMap: () => JSON.parse(JSON.stringify(unwired)),
     slots: () => slots,

app/legacy/pipeline_state.test.js

@@ -239,3 +239,26 @@ test("a refused restore leaves the pipeline as it is", async () => {
   assert.deepStrictEqual(res.refused, ["no"]);
   assert.strictEqual(PS.slots().find((s) => s.id === "model").inputs.file, "changed");
 });
+
+test("a module switched off is remembered with the settings, listed as off, and can be switched back on", async () => {
+  const posts = [];
+  const PS = load(posts);
+  await PS.ensureLoaded();
+  global.window.MovieEditorAPI.modules = async () => ({ modules: [{ id: "sharpen", settings: {} }, { id: "alg", settings: {} }], control: {} });
+  await PS.refreshControl();
+  await PS.setOff("sharpen", true);
+  assert.strictEqual(PS.isOff("sharpen"), true);
+  assert.strictEqual(PS.isOff("alg"), false);
+  assert.deepStrictEqual(posts.at(-1).values._off, { modules: ["sharpen"] });
+  await PS.setOff("alg", true);
+  await PS.setOff("sharpen", false);
+  assert.deepStrictEqual(posts.at(-1).values._off, { modules: ["alg"] });
+});
+
+test("a project opened after another does not inherit the first one's pending switch", async () => {
+  const PS = load([]);
+  await PS.ensureLoaded();
+  await PS.setOff("sharpen", true);
+  await PS.adopt([{ id: "model", node: "Loader", inputs: { file: "x" } }]);
+  assert.strictEqual(PS.isOff("sharpen"), false);
+});

app/legacy/sampler_quickbar.js

@@ -29,7 +29,7 @@
   }
 
   function samplingModules() {
-    return Object.values(PS.modulesById())
+    return PS.activeModules()
       .filter((m) => (m.category || "") === "sampling" && Object.keys(m.settings || {}).length);
   }
 

app/legacy/store.js

@@ -3684,6 +3684,7 @@
         const mediaList = [...(images || []), ...(audio || [])]
           .map((f) => ({ ...f, kind: _kindForFilename(f.filename) }));
         if (mediaList.length && _genRunSceneIds.length) _recordSegment(mediaList, _genRunSceneIds, { promptId });
+        _announceQuarantine();
         if (getEditorSetting("upscaleMode") === "always") upscaleMedia(mediaList.find((m) => m.kind === "videos"));
         set({
           gen: {
@@ -3692,6 +3693,7 @@
           },
         });
       } else if (phase === "failed") {
+        _announceQuarantine();
         set({
           gen: {
             state: "error", promptId, media: [],
@@ -4695,6 +4697,21 @@
     }
   }
 
+  // A module that failed during a run is now off for good (until repaired): say which, once.
+  const _quarantineSeen = new Set();
+  async function _announceQuarantine() {
+    const PS = window.PipelineState;
+    if (!PS || !PS.refreshControl) return;
+    const control = await PS.refreshControl();
+    const fresh = Object.entries(control).filter(([id, c]) => c.quarantine && !_quarantineSeen.has(id));
+    if (!fresh.length) return;
+    fresh.forEach(([id]) => _quarantineSeen.add(id));
+    const title = (id) => (PS.modulesById()[id] || {}).title || id;
+    state.notice = fresh.map(([id, c]) => `${title(id)} failed and is now off (${c.quarantine.reason}). `
+      + `Turn it back on in Settings ▸ Modules once it is repaired.`).join(" ");
+    notify();
+  }
+
   // ── pluggable models / exposed controls ──────────────────────────────────────
   // The pipeline lives in PipelineState (one copy shared by every panel); the project keeps a
   // copy of it (project.models) so a model pick and every node value travel with the project to

conftest.py

@@ -159,3 +159,10 @@ def packed_av(video, audio):
 def unpacked(x, latent_shapes):
     import comfy.utils
     return comfy.utils.unpack_latents(x, latent_shapes)
+
+
+@pytest.fixture(autouse=True)
+def _quarantine_in_tmp(tmp_path, monkeypatch):
+    """A test that makes a module fail must never write the real quarantine file."""
+    from core import config
+    monkeypatch.setattr(config, "QUARANTINE_FILE", tmp_path / "quarantine.json")

core/chain.py

@@ -115,6 +115,8 @@ def build(specs, values, accepts: Sequence[str], dropped: patching.Dropped,
             member = make(values.get(spec.id) or {}, tuple(offered))
         except Exception as exc:                 # noqa: BLE001
             log.broke(f"{spec.id}.{capability}", exc, "starting up for this run")
+            from . import control
+            control.start_failed(spec, exc)
             notes.append(f"{spec.id}: failed to start -- {type(exc).__name__}: {exc}")
             continue
 

core/config.py

@@ -39,3 +39,6 @@ SHORTCUT_CATEGORIES_FILE = ROOT / "shortcut_categories.json"
 # The words that cut a story into scenes (core/story.py). Global for the same
 # reason the shortcut library is.
 MARKERS_FILE = ROOT / "markers.json"
+
+# Modules that failed and stay off until repaired (core/control.py). State, so beside the library.
+QUARANTINE_FILE = ROOT / "quarantine.json"

core/control.py

@@ -0,0 +1,193 @@
+"""Which modules run: the project's own switch, and a module's persistent quarantine.
+
+A module that raises mid-run is dropped for that run (core/patching.py). That alone leaves a
+faulty module to fail again on every generation, so a fault is also written down here and the
+module stays OFF until it is repaired -- its code changed -- or the person turns it back on.
+The project's switch is the person's own: a module they turned off vanishes for that project.
+
+Only modules that MODIFY a run can be switched (they provide `modifier` or `sampler_modifier`).
+Loaders, the sampler and the like are structure: a pipeline without them is not a pipeline.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import threading
+import time
+from pathlib import Path
+from typing import Dict, Iterable, List, Tuple
+
+from . import config, log, patching
+
+#: Keys of the settings payload that are not module ids.
+RESERVED = frozenset({"_off"})
+
+_TRANSIENT = ("Interrupt", "OutOfMemory", "OOM")        # not the module's fault: say nothing, remember nothing
+_lock = threading.Lock()
+
+
+def controllable(spec) -> bool:
+    return bool(spec.provides.get("modifier") or spec.provides.get("sampler_modifier"))
+
+
+def off_by_project(settings) -> set:
+    """Module ids the project turned off, read from the settings payload."""
+    raw = settings.get("_off") if isinstance(settings, dict) else None
+    ids = raw.get("modules") if isinstance(raw, dict) else None
+    return {i for i in ids if isinstance(i, str)} if isinstance(ids, list) else set()
+
+
+def bad_off(raw) -> str | None:
+    """Why a payload's `_off` entry is malformed, or None."""
+    if raw is None:
+        return None
+    ids = raw.get("modules") if isinstance(raw, dict) else None
+    if not isinstance(ids, list) or not all(isinstance(i, str) for i in ids):
+        return '"_off" must be {"modules": ["module_id", ...]}'
+    return None
+
+
+# --- the quarantine file ---------------------------------------------------
+
+
+def _file() -> Path:
+    return Path(config.QUARANTINE_FILE)
+
+
+def _read() -> Dict[str, dict]:
+    try:
+        raw = json.loads(_file().read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+    return {k: v for k, v in raw.items() if isinstance(k, str) and isinstance(v, dict)} if isinstance(raw, dict) else {}
+
+
+def _write(entries: Dict[str, dict]) -> None:
+    path = _file()
+    tmp = path.with_suffix(".tmp")
+    try:
+        if not entries:
+            path.unlink(missing_ok=True)
+            return
+        tmp.write_text(json.dumps(entries, indent=1), encoding="utf-8")
+        os.replace(tmp, path)
+    except OSError as exc:
+        log.warning("module control", f"could not write {path.name}: {exc}. The module is off for this "
+                                      f"session only.")
+
+
+def signature(spec) -> str:
+    """Changes when the module's code does: that is what "repaired" means."""
+    mod = sys.modules.get(spec.source)
+    base = Path(getattr(mod, "__file__", "") or "").parent
+    parts = []
+    if base.is_dir():
+        for p in sorted(base.rglob("*.py")):
+            if "tests" in p.parts or "__pycache__" in p.parts:
+                continue
+            try:
+                st = p.stat()
+            except OSError:
+                continue
+            parts.append(f"{p.name}:{st.st_mtime_ns}:{st.st_size}")
+    return "|".join(parts)
+
+
+def quarantined(specs: Iterable = ()) -> Dict[str, dict]:
+    """{module id: {reason, when}} for modules still quarantined. One whose code has changed since
+    it failed is released here, and said."""
+    by_id = {s.id: s for s in specs}
+    with _lock:
+        entries = _read()
+        kept = {}
+        for mid, entry in entries.items():
+            spec = by_id.get(mid)
+            if spec is not None and entry.get("sig") and entry["sig"] != signature(spec):
+                log.info("module control", f"{mid} was changed since it failed: trying it again.")
+                continue
+            kept[mid] = entry
+        if kept != entries:
+            _write(kept)
+    return kept
+
+
+def quarantine(spec, reason: str) -> None:
+    with _lock:
+        entries = _read()
+        if spec.id in entries:
+            return
+        entries[spec.id] = {"reason": str(reason)[:400], "when": time.strftime("%Y-%m-%d %H:%M"),
+                            "sig": signature(spec)}
+        _write(entries)
+    log.alert("module control", f"{spec.id} failed and is now OFF until it is repaired or you turn it "
+                                f"back on (Settings ▸ Modules): {reason}")
+
+
+def release(module_id: str) -> bool:
+    with _lock:
+        entries = _read()
+        if module_id not in entries:
+            return False
+        del entries[module_id]
+        _write(entries)
+    return True
+
+
+def fault(key: str, exc: BaseException) -> None:
+    """Called by core/patching.Dropped when a module's hook raised. Faults that are not the module's
+    (an interrupt, running out of memory) are left alone."""
+    if any(t in type(exc).__name__ for t in _TRANSIENT):
+        return
+    mid = key[len("funpack."):].split(".")[0] if key.startswith("funpack.") else None
+    if not mid:
+        return
+    from . import registry
+    spec = registry.current().specs.get(mid)
+    if spec is not None and controllable(spec):
+        quarantine(spec, f"{type(exc).__name__}: {exc}")
+
+
+patching.on_fault = fault
+
+
+def start_failed(spec, exc: BaseException) -> None:
+    """A module that raised while installing or starting up for a run."""
+    if controllable(spec) and not any(t in type(exc).__name__ for t in _TRANSIENT):
+        quarantine(spec, f"{type(exc).__name__}: {exc}")
+
+
+# --- what a run uses -------------------------------------------------------
+
+
+def partition(specs: Iterable, settings) -> Tuple[List, List[str]]:
+    """(kept, notes): `specs` without the ones the project turned off or that are quarantined."""
+    specs = list(specs)
+    off = off_by_project(settings)
+    held = quarantined(specs)
+    kept, notes = [], []
+    for spec in specs:
+        if controllable(spec):
+            if spec.id in off:
+                notes.append(f"{spec.id}: turned off for this project")
+                continue
+            if spec.id in held:
+                notes.append(f"{spec.id}: OFF -- it failed on {held[spec.id].get('when', '?')} "
+                             f"({held[spec.id].get('reason', '')}). Turn it back on in Settings ▸ Modules.")
+                continue
+        kept.append(spec)
+    return kept, notes
+
+
+def fingerprint() -> str:
+    """Part of a node's cache key: a module going into or out of quarantine changes what a run does."""
+    return ",".join(sorted(_read()))
+
+
+def state(specs: Iterable) -> Dict[str, dict]:
+    """What the Modules panel shows: {id: {controllable, quarantine?}} for every module."""
+    specs = list(specs)
+    held = quarantined(specs)
+    return {s.id: {"controllable": controllable(s), **({"quarantine": held[s.id]} if s.id in held else {})}
+            for s in specs}

core/patching.py

@@ -111,6 +111,10 @@ def _strip_options(options: dict, prefix: str) -> int:
 # this" and "this happened", so it is said once, with the traceback, and carried
 # in the run's status.
 
+#: Set by core/control.py: told when a module's hook raised, so the fault outlives the run.
+on_fault = None
+
+
 class Dropped:
     """What was disabled during a run, and why. One per run, not per process."""
 
@@ -118,9 +122,10 @@ class Dropped:
         self.reasons: dict = {}
 
     def clear(self) -> None:
-        """Forget what was dropped. Called when a generation starts, because a
-        modifier that failed on the last one deserves its chance at this one --
-        and because ComfyUI's cache hands the same object to both."""
+        """Forget what was dropped. Called when a generation starts: this run's record starts
+        empty (a module that FAILED also stays off across runs: that is
+        core/control.py's quarantine, not this record) -- and because ComfyUI's cache hands the same
+        object to both."""
         self.reasons.clear()
 
     def record(self, key: str, exc: BaseException) -> bool:
@@ -128,6 +133,11 @@ class Dropped:
         if key in self.reasons:
             return False
         self.reasons[key] = f"{type(exc).__name__}: {exc}"
+        if on_fault is not None:
+            try:
+                on_fault(key, exc)
+            except Exception:                    # noqa: BLE001 -- bookkeeping must not end a run
+                pass
         return True
 
     def __contains__(self, key) -> bool:

core/routes.py

@@ -8,7 +8,7 @@ thin adapters over pure functions in `serve`.
 import asyncio
 import json
 
-from . import (backend_log, config, graph as graph_mod, log, media, nodes_manager, probe as probe_mod,
+from . import (backend_log, config, control as control_mod, graph as graph_mod, log, media, nodes_manager, probe as probe_mod,
                projects,
                prompt_build,
                settings_card,
@@ -141,6 +141,8 @@ def manifest(traits=None):
         "incompatible": [
             {"id": spec.id, "requires": spec.requires} for spec in incompatible
         ],
+        # Which modules the person can switch, and which are quarantined after a fault.
+        "control": control_mod.state(ordered),
     }
 
 
@@ -173,6 +175,18 @@ def register(routes, prefix=None):
         traits = [t for t in raw.split(",") if t] if raw is not None else None
         return web.json_response(manifest(traits))
 
+    @routes.post(P + "/api/control/release")
+    async def _control_release(req):
+        """Turn a quarantined module back on: {"id": "<module id>"}."""
+        try:
+            body = await req.json()
+        except ValueError:
+            body = None
+        mid = body.get("id") if isinstance(body, dict) else None
+        if not isinstance(mid, str) or mid not in modules().specs:
+            return web.json_response({"detail": "Name a module that is installed."}, status=400)
+        return web.json_response({"released": control_mod.release(mid)})
+
     def _pipeline():
         """Whatever module offers a default pipeline, or nothing.
 

core/tests/test_control.py

@@ -0,0 +1,73 @@
+"""Module control: the project's switch and the persistent quarantine."""
+
+import types
+
+from core import control, patching
+
+
+def spec(mid, provides=("modifier",), source="nowhere"):
+    return types.SimpleNamespace(id=mid, provides={p: (lambda *a, **k: None) for p in provides}, source=source)
+
+
+def test_only_modules_that_modify_a_run_can_be_switched():
+    assert control.controllable(spec("a", ("modifier",)))
+    assert control.controllable(spec("a", ("sampler_modifier",)))
+    assert not control.controllable(spec("loader", ("routes",)))
+
+
+def test_a_module_the_project_turned_off_is_left_out_and_said():
+    kept, notes = control.partition([spec("a"), spec("b")], {"_off": {"modules": ["a"]}})
+    assert [s.id for s in kept] == ["b"] and "a: turned off for this project" in notes[0]
+
+
+def test_structure_cannot_be_turned_off():
+    kept, _ = control.partition([spec("loader", ("routes",))], {"_off": {"modules": ["loader"]}})
+    assert [s.id for s in kept] == ["loader"]
+
+
+def test_a_fault_is_remembered_across_runs_until_released():
+    a = spec("a")
+    control.quarantine(a, "RuntimeError: boom")
+    kept, notes = control.partition([a], {})
+    assert kept == [] and "a: OFF" in notes[0] and "boom" in notes[0] and "Settings ▸ Modules" in notes[0]
+    assert control.fingerprint() == "a"
+    assert control.release("a") and not control.release("a")
+    assert [s.id for s in control.partition([a], {})[0]] == ["a"]
+
+
+def test_a_module_whose_code_changed_since_it_failed_gets_another_try(monkeypatch):
+    a = spec("a")
+    monkeypatch.setattr(control, "signature", lambda s: "v1")
+    control.quarantine(a, "boom")
+    monkeypatch.setattr(control, "signature", lambda s: "v2")
+    assert [s.id for s in control.partition([a], {})[0]] == ["a"]
+    assert control.fingerprint() == ""
+
+
+def test_an_interrupt_or_out_of_memory_is_not_the_modules_fault(monkeypatch):
+    from core import registry
+    a = spec("a")
+    monkeypatch.setattr(registry, "current", lambda: types.SimpleNamespace(specs={"a": a}))
+
+    class InterruptProcessingException(Exception): ...
+    class OutOfMemoryError(Exception): ...
+
+    control.fault("funpack.a", InterruptProcessingException())
+    control.fault("funpack.a", OutOfMemoryError())
+    assert control.fingerprint() == ""
+    control.fault("funpack.a", ValueError("bad"))
+    assert control.fingerprint() == "a"
+
+
+def test_a_hook_that_raised_in_a_run_reaches_the_quarantine(monkeypatch):
+    from core import registry
+    a = spec("a")
+    monkeypatch.setattr(registry, "current", lambda: types.SimpleNamespace(specs={"a": a}))
+    dropped = patching.Dropped()
+    assert dropped.record("funpack.a", RuntimeError("step 3"))
+    assert "a" in control.quarantined([a]) and "step 3" in control.quarantined([a])["a"]["reason"]
+
+
+def test_a_malformed_off_entry_is_named():
+    assert control.bad_off({"modules": ["a"]}) is None and control.bad_off(None) is None
+    assert "modules" in control.bad_off(["a"]) and "modules" in control.bad_off({"modules": [1]})

core/tests/test_routes_pipeline.py

@@ -515,3 +515,19 @@ def test_a_module_provided_preset_is_offered(server, registered):
     preset = next(p for p in body["presets"] if p["id"] == "minimax_h3_reference_to_video")
     assert preset["module"] == "model_minimax_h3"
     assert preset["slots"]
+
+
+def test_the_manifest_says_which_modules_can_be_switched_and_a_quarantined_one_can_be_released(server):
+    from core import control, registry
+    status, body = _request(server, "GET", "/funpack/api/modules")
+    assert status == 200 and body["control"]
+    sharp = registry.current().specs["sharpen"]
+    assert body["control"]["sharpen"]["controllable"] is True
+    control.quarantine(sharp, "RuntimeError: boom")
+    _, body = _request(server, "GET", "/funpack/api/modules")
+    assert body["control"]["sharpen"]["quarantine"]["reason"].endswith("boom")
+    status, body = _request(server, "POST", "/funpack/api/control/release", {"id": "sharpen"})
+    assert (status, body) == (200, {"released": True})
+    _, body = _request(server, "GET", "/funpack/api/modules")
+    assert "quarantine" not in body["control"]["sharpen"]
+    assert _request(server, "POST", "/funpack/api/control/release", {"id": "nope"})[0] == 400

modules/_core.py

@@ -11,8 +11,8 @@ leading underscore keeps it out of the module scan.
 """
 
 try:                                             # root on sys.path: tests, dev server
-    from core import chain, config, contract, dit_hooks, input_steer, log, media, patching, projects, rated_dial, registry, relations, run, schema, shortcuts, streams, temp_files, traits
+    from core import chain, config, contract, control, dit_hooks, input_steer, log, media, patching, projects, rated_dial, registry, relations, run, schema, shortcuts, streams, temp_files, traits
 except ImportError:                              # inside ComfyUI: a subpackage of the pack
-    from ..core import chain, config, contract, dit_hooks, input_steer, log, media, patching, projects, rated_dial, registry, relations, run, schema, shortcuts, streams, temp_files, traits
+    from ..core import chain, config, contract, control, dit_hooks, input_steer, log, media, patching, projects, rated_dial, registry, relations, run, schema, shortcuts, streams, temp_files, traits
 
-__all__ = ["chain", "config", "contract", "dit_hooks", "input_steer", "log", "media", "patching", "projects", "rated_dial", "registry", "relations", "run", "schema", "shortcuts", "streams", "temp_files", "traits"]
+__all__ = ["chain", "config", "contract", "control", "dit_hooks", "input_steer", "log", "media", "patching", "projects", "rated_dial", "registry", "relations", "run", "schema", "shortcuts", "streams", "temp_files", "traits"]

modules/sampling/modifiers/nodes.py

@@ -33,7 +33,7 @@ import json
 
 from comfy_api.latest import io
 
-from ..._core import (log, patching, registry as registry_mod,
+from ..._core import (control as control_mod, log, patching, registry as registry_mod,
                       relations as relations_mod, run as run_mod,
                       schema as schema_mod)
 
@@ -120,6 +120,11 @@ class FunPackModifierSettings(io.ComfyNode):
         specs = registry_mod.current().specs
         problems = []
         for module_id, values in raw.items():
+            if module_id in control_mod.RESERVED:
+                said = control_mod.bad_off(values)
+                if said:
+                    problems.append(said)
+                continue
             spec = specs.get(module_id)
             if spec is None:
                 problems.append(f"no module named {module_id!r} is installed")
@@ -152,6 +157,13 @@ class FunPackModifierSettings(io.ComfyNode):
         guards_off = cls._guards_off(raw)
         problems, checked = [], {}
         for module_id, values in raw.items():
+            if module_id in control_mod.RESERVED:
+                said = control_mod.bad_off(values)
+                if said:
+                    problems.append(said)
+                else:
+                    checked[module_id] = values
+                continue
             spec = specs.get(module_id)
             if spec is None:
                 # Silently ignoring it is how v4 accumulated settings that meant
@@ -192,8 +204,9 @@ class FunPackLoadModifiers(io.ComfyNode):
 
     @classmethod
     def fingerprint_inputs(cls, **_):
-        return "|".join(f"{spec.id}={answer()}"
-                        for spec, answer in registry_mod.current().providers(CACHE_KEY))
+        return "|".join([f"{spec.id}={answer()}"
+                         for spec, answer in registry_mod.current().providers(CACHE_KEY)]
+                        + ["quarantine=" + control_mod.fingerprint()])
 
     @classmethod
     def execute(cls, model, settings=None) -> io.NodeOutput:
@@ -214,6 +227,7 @@ class FunPackLoadModifiers(io.ComfyNode):
 
         offering = [spec for spec in specs if spec.provides.get(CAPABILITY)]
         compatible, incompatible = traits_mod.split(offering, available)
+        compatible, switched = control_mod.partition(compatible, settings)
         ordered, rejected = order(compatible)
 
         patched = model.clone()
@@ -241,6 +255,7 @@ class FunPackLoadModifiers(io.ComfyNode):
                          "these settings are yours.")
         if stripped:
             notes.append(f"replaced {stripped} modifier(s) already on this model")
+        notes.extend(switched)
 
         # Checked HERE, not only where the payload was made. A FUNPACK_SETTINGS
         # socket is typed by a string tag, so a dict can reach this node without
@@ -270,6 +285,7 @@ class FunPackLoadModifiers(io.ComfyNode):
                 # Absent, and said out loud. A modifier that half-installed and
                 # carried on is how a run silently stops meaning what it says.
                 log.broke(f"{spec.id}.{CAPABILITY}", exc, "installing itself")
+                control_mod.start_failed(spec, exc)
                 notes.append(f"{spec.id}: failed to install -- {type(exc).__name__}: {exc}")
                 continue
             if note is None:

modules/sampling/sampler/nodes.py

@@ -27,7 +27,7 @@ import torch
 from comfy_api.latest import io
 
 from ..._core import chain as chain_mod, log, patching, registry as registry_mod, run as run_mod
-from ..._core import relations as relations_mod, schema as schema_mod, traits as traits_mod
+from ..._core import control as control_mod, relations as relations_mod, schema as schema_mod, traits as traits_mod
 
 # The hook points this sampler offers. A modifier asking for anything else is
 # absent and said, rather than half-running.
@@ -201,6 +201,7 @@ class FunPackSampler(io.ComfyNode):
 
         offering = [s for s in specs if s.provides.get("sampler_modifier")]
         compatible, incompatible = traits_mod.split(offering, available)
+        compatible, switched = control_mod.partition(compatible, settings)
         ordered, rejected = relations_mod.order(compatible)
 
         guards_off = any(answer(settings or {}) for _spec, answer
@@ -218,7 +219,7 @@ class FunPackSampler(io.ComfyNode):
             notes.append(f"{spec.id}: needs {', '.join(traits_mod.missing_for(spec, available))}")
         for spec, why in rejected:
             notes.append(f"{spec.id}: {why}")
-        return chain, notes
+        return chain, notes + switched
 
     @classmethod
     def _with_chain(cls, sampler, chain, sigmas, run):

modules/sampling/tests/test_a_module_cannot_kill_a_run.py

@@ -136,11 +136,32 @@ def test_the_record_is_per_run_not_per_process(patcher):
         hook(_pre_cfg_args([torch.randn(1, 4, 8, 16, 16), torch.randn(1, 4, 8, 16, 16)]))
         return patched.funpack_dropped
 
-    first, second = run(), run()
+    from core import control
+    first = run()
+    assert control.release("sharpen")            # a fault is also remembered across runs: see core/tests/test_control.py
+    second = run()
     assert first is not second
     assert "funpack.sharpen" in first and "funpack.sharpen" in second
 
 
+def test_a_module_that_failed_stays_off_for_the_next_generation_until_turned_back_on(patcher):
+    import torch
+    from core import control
+    from modules.sampling.modifiers.nodes import FunPackLoadModifiers
+
+    values = {"sharpen": {"enabled": True, "amount": 0.4}}
+    first, _ = FunPackLoadModifiers.execute(patcher, settings=values).result
+    first.model_options["sampler_pre_cfg_function"][0](
+        _pre_cfg_args([torch.randn(1, 4, 8, 16, 16), torch.randn(1, 4, 8, 16, 16)]))
+    assert "sharpen" in control.quarantined(__import__("core.registry", fromlist=["x"]).current().specs.values())
+    second, status = FunPackLoadModifiers.execute(patcher, settings=values).result
+    assert "sampler_pre_cfg_function" not in second.model_options
+    assert "sharpen: OFF" in status and "Settings ▸ Modules" in status
+    control.release("sharpen")
+    third, _ = FunPackLoadModifiers.execute(patcher, settings=values).result
+    assert "sampler_pre_cfg_function" in third.model_options
+
+
 def test_a_hook_shape_with_no_known_neutral_result_is_named(patcher):
     """Honesty about the limit: an unguardable hook is installed and said out
     loud, rather than counted as protected."""
@@ -207,6 +228,8 @@ def test_the_guards_are_on_unless_asked_otherwise(patcher):
         original = video[0].clone()
         out = hook(_pre_cfg_args(video))          # must not raise
         assert torch.equal(out[0], original)
+        from core import control
+        control.release("sharpen")                # it faulted on purpose; each case starts with it on
 
 
 def test_is_on_reads_the_switch_the_same_way_everywhere():

modules/sampling/tests/test_modifier_settings.py

@@ -156,3 +156,13 @@ def test_validation_catches_the_same_things_execution_would(registry, payload, e
 
     with pytest.raises(RuntimeError):
         _run(payload)
+
+
+def test_the_projects_off_list_passes_through_and_is_not_mistaken_for_a_module(registry):
+    out, _status = _run('{"alg": {"enabled": true}, "_off": {"modules": ["sharpen"]}}')
+    assert out["_off"] == {"modules": ["sharpen"]} and out["alg"]["enabled"] is True
+
+
+def test_a_malformed_off_list_is_refused_with_a_reason(registry):
+    with pytest.raises(RuntimeError, match="modules"):
+        _run('{"_off": ["sharpen"]}')