fix(config): report settings persistence failures (#220)
Settings saves now go through one serialized config store (py/config_store.py). A commit re-reads config.json under a process lock, applies the change to a copy and replaces the file atomically (temp file beside it, fsync, close, os.replace), so a failed write leaves the old file in place. POST /prompt_manager/settings commits first and only then publishes the values at runtime. A write failure is 500 config-write-failed, an unparseable config.json is 409 config-unreadable (the file is kept instead of replaced) and a stale expected_revision is 409 settings-changed. A save reports persisted, revision, runtime_applied and restart_required_fields; GET adds the revision. Only submitted keys are written; other sections and the CivitAI key are kept.
This commit is contained in:
Vito Sansevero
+146
-54
@@ -26,6 +26,12 @@ from .jobs import (
|
||||
JobBusy,
|
||||
)
|
||||
|
||||
from ..config_store import (
|
||||
ConfigUnreadable,
|
||||
RevisionConflict,
|
||||
get_config_store,
|
||||
)
|
||||
|
||||
try:
|
||||
from ...utils.parallel import map_parallel
|
||||
from ...utils.validators import validate_result_timeout, validate_worker_threads
|
||||
@@ -133,6 +139,45 @@ def _thumbnail_rel_path(rel_path, thumbnail_ext):
|
||||
return f"thumbnails/{rel_path.with_suffix('').as_posix()}_thumb{thumbnail_ext}"
|
||||
|
||||
|
||||
# Settings owned by POST /settings: config attribute -> (section, key).
|
||||
_SETTINGS_FILE_KEYS = {
|
||||
"RESULT_TIMEOUT": ("web_ui", "result_timeout"),
|
||||
"WEBUI_DISPLAY_MODE": ("web_ui", "webui_display_mode"),
|
||||
"INFINITE_SCROLL": ("web_ui", "infinite_scroll"),
|
||||
"WORKER_THREADS": ("performance", "worker_threads"),
|
||||
}
|
||||
|
||||
|
||||
def _section(config, name):
|
||||
"""``config[name]`` as a fresh dict (a non-object section starts empty)."""
|
||||
value = config.get(name)
|
||||
return dict(value) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _settings_mutation(updates, new_roots):
|
||||
"""The config commit for a settings save: only the submitted keys change.
|
||||
|
||||
Every other section and key (integrations and the CivitAI key, database,
|
||||
unknown sections) is kept as it is on disk.
|
||||
"""
|
||||
|
||||
def mutate(config):
|
||||
for name, value in updates.items():
|
||||
section, key = _SETTINGS_FILE_KEYS[name]
|
||||
merged = _section(config, section)
|
||||
merged[key] = value
|
||||
config[section] = merged
|
||||
if new_roots is not None:
|
||||
gallery = _section(config, "gallery")
|
||||
monitoring = _section(gallery, "monitoring")
|
||||
monitoring["directories"] = list(new_roots)
|
||||
gallery["monitoring"] = monitoring
|
||||
config["gallery"] = gallery
|
||||
return config
|
||||
|
||||
return mutate
|
||||
|
||||
|
||||
async def _read_json(request):
|
||||
"""Parse a JSON object body through the package's capped reader.
|
||||
|
||||
@@ -890,9 +935,11 @@ class AdminRoutesMixin:
|
||||
root_paths = [
|
||||
self._public_path(d) for d in GalleryConfig.MONITORING_DIRECTORIES
|
||||
]
|
||||
revision = await self._run_in_executor(self._config_revision)
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
**revision,
|
||||
"settings": {
|
||||
"result_timeout": PromptManagerConfig.RESULT_TIMEOUT,
|
||||
"webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE,
|
||||
@@ -924,14 +971,25 @@ class AdminRoutesMixin:
|
||||
return list(GalleryConfig.MONITORING_DIRECTORIES)
|
||||
|
||||
async def save_settings(self, request):
|
||||
"""Save settings: the whole payload is validated before anything changes."""
|
||||
try:
|
||||
from ..config import PromptManagerConfig
|
||||
"""Save settings: validate, commit to config.json, then publish (#220).
|
||||
|
||||
Nothing changes at runtime until config.json holds the new values, so
|
||||
a save that cannot be written reports ``persisted: false`` and leaves
|
||||
the running settings as they were. ``expected_revision`` (the
|
||||
``revision`` from GET) refuses to overwrite a file changed meanwhile;
|
||||
without it the payload merges into what is on disk (legacy page).
|
||||
"""
|
||||
try:
|
||||
data, error_response = await _read_json(request)
|
||||
if error_response is not None:
|
||||
return error_response
|
||||
|
||||
expected_revision = data.get("expected_revision")
|
||||
if expected_revision is not None and not isinstance(expected_revision, str):
|
||||
return web.json_response(
|
||||
{"success": False, "error": "expected_revision must be a string"},
|
||||
status=400,
|
||||
)
|
||||
try:
|
||||
updates = self._validate_settings(data)
|
||||
except ValueError as ve:
|
||||
@@ -943,19 +1001,20 @@ class AdminRoutesMixin:
|
||||
if error:
|
||||
return web.json_response({"success": False, "error": error}, status=400)
|
||||
|
||||
for name, value in updates.items():
|
||||
setattr(PromptManagerConfig, name, value)
|
||||
restart_required = False
|
||||
if new_roots is not None:
|
||||
restart_required = self._apply_gallery_roots(new_roots)
|
||||
|
||||
self._persist_settings()
|
||||
try:
|
||||
_saved, revision = await self._run_in_executor(
|
||||
get_config_store().commit,
|
||||
_settings_mutation(updates, new_roots),
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
except (ConfigUnreadable, RevisionConflict, OSError) as exc:
|
||||
return self._config_commit_failure(exc)
|
||||
|
||||
return web.json_response(
|
||||
{
|
||||
"success": True,
|
||||
"message": "Settings saved successfully",
|
||||
"restart_required": restart_required,
|
||||
**self._publish_settings(updates, new_roots, revision),
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -965,6 +1024,82 @@ class AdminRoutesMixin:
|
||||
status=500,
|
||||
)
|
||||
|
||||
def _publish_settings(self, updates, new_roots, revision):
|
||||
"""Install committed settings at runtime; the ConfigCommit fields.
|
||||
|
||||
The values are already durable. A publication that fails is reported
|
||||
as ``runtime_applied: false`` with the fields that need a restart;
|
||||
nothing is retried or rolled back.
|
||||
"""
|
||||
from ..config import PromptManagerConfig
|
||||
|
||||
for name, value in updates.items():
|
||||
setattr(PromptManagerConfig, name, value)
|
||||
restart_fields = []
|
||||
runtime_applied = True
|
||||
if new_roots is not None:
|
||||
try:
|
||||
if self._apply_gallery_roots(new_roots):
|
||||
restart_fields.append("gallery_root_paths")
|
||||
except Exception as e:
|
||||
self.logger.error(
|
||||
f"Saved gallery roots could not be applied: {e}", exc_info=True
|
||||
)
|
||||
runtime_applied = False
|
||||
restart_fields.append("gallery_root_paths")
|
||||
return {
|
||||
"restart_required": bool(restart_fields),
|
||||
"persisted": True,
|
||||
"revision": revision,
|
||||
"runtime_applied": runtime_applied,
|
||||
"restart_required_fields": restart_fields,
|
||||
}
|
||||
|
||||
def _config_commit_failure(self, exc):
|
||||
"""The response for a config commit that wrote nothing."""
|
||||
not_saved = {"success": False, "persisted": False, "runtime_applied": False}
|
||||
if isinstance(exc, ConfigUnreadable):
|
||||
self.logger.warning(f"Settings not saved: {exc}")
|
||||
return web.json_response(
|
||||
{
|
||||
**not_saved,
|
||||
"code": "config-unreadable",
|
||||
"error": (
|
||||
"config.json cannot be read, so nothing was saved. "
|
||||
"Fix or remove the file, then save again."
|
||||
),
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
if isinstance(exc, RevisionConflict):
|
||||
return web.json_response(
|
||||
{
|
||||
**not_saved,
|
||||
"code": "settings-changed",
|
||||
"error": "Settings changed since they were loaded. Reload them.",
|
||||
"revision": exc.current,
|
||||
},
|
||||
status=409,
|
||||
)
|
||||
self.logger.error(f"Could not write config.json: {exc}")
|
||||
return web.json_response(
|
||||
{
|
||||
**not_saved,
|
||||
"code": "config-write-failed",
|
||||
"error": "Settings could not be written to config.json.",
|
||||
},
|
||||
status=500,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _config_revision():
|
||||
"""``{"revision": ...}``, or a null revision with ``config_error``."""
|
||||
try:
|
||||
_data, revision = get_config_store().read()
|
||||
except ConfigUnreadable:
|
||||
return {"revision": None, "config_error": "config-unreadable"}
|
||||
return {"revision": revision}
|
||||
|
||||
@staticmethod
|
||||
def _validate_settings(data):
|
||||
"""Scalar settings of a payload as ``{CONFIG_ATTR: value}``.
|
||||
@@ -1041,49 +1176,6 @@ class AdminRoutesMixin:
|
||||
self.reset_gallery_roots_cache()
|
||||
return True
|
||||
|
||||
def _persist_settings(self):
|
||||
"""Merge the user-editable settings into the configured config.json.
|
||||
|
||||
Only the keys this endpoint owns are rewritten; every other section
|
||||
(integrations, database, performance, ...) and every other key in
|
||||
``web_ui``/``gallery`` is kept as it was. A config file that cannot
|
||||
be parsed is replaced.
|
||||
"""
|
||||
from ..config import GalleryConfig, PromptManagerConfig, write_private_json
|
||||
|
||||
config_file = PromptManagerConfig.get_config_path()
|
||||
existing = self._read_config_file(config_file)
|
||||
web_ui = dict(existing.get("web_ui") or {})
|
||||
web_ui["result_timeout"] = PromptManagerConfig.RESULT_TIMEOUT
|
||||
web_ui["webui_display_mode"] = PromptManagerConfig.WEBUI_DISPLAY_MODE
|
||||
web_ui["infinite_scroll"] = PromptManagerConfig.INFINITE_SCROLL
|
||||
gallery = dict(existing.get("gallery") or {})
|
||||
monitoring = dict(gallery.get("monitoring") or {})
|
||||
monitoring["directories"] = list(GalleryConfig.MONITORING_DIRECTORIES)
|
||||
gallery["monitoring"] = monitoring
|
||||
performance = dict(existing.get("performance") or {})
|
||||
performance["worker_threads"] = PromptManagerConfig.WORKER_THREADS
|
||||
config_data = {
|
||||
**existing,
|
||||
"web_ui": web_ui,
|
||||
"gallery": gallery,
|
||||
"performance": performance,
|
||||
}
|
||||
try:
|
||||
write_private_json(config_file, config_data)
|
||||
self.logger.info(f"Settings saved to {config_file}")
|
||||
except OSError as save_err:
|
||||
self.logger.warning(f"Could not save config file: {save_err}")
|
||||
|
||||
def _read_config_file(self, config_file):
|
||||
"""Current contents of config.json as a dict, or {} when unusable."""
|
||||
try:
|
||||
with open(config_file, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
async def run_diagnostics(self, request):
|
||||
"""Run comprehensive system diagnostics and health checks."""
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""The one serialized writer of config.json (#220).
|
||||
|
||||
Every save that changes config.json goes through ``ConfigStore.commit``: the
|
||||
settings page, the LoRA integration and, later, every System section. A
|
||||
commit runs under one process-wide lock, re-reads the file inside it, applies
|
||||
the caller's change to a copy and replaces the file atomically. So two saves
|
||||
never overwrite each other's keys, sections this process does not know are
|
||||
kept byte-for-byte in meaning, and a write that fails leaves the old file in
|
||||
place and raises instead of being logged and reported as saved.
|
||||
|
||||
A revision names the file content a client last saw: ``"sha256:<hex>"`` of
|
||||
the file bytes, or ``"absent"`` when there is no file. A commit that names a
|
||||
revision refuses to write over different content (``RevisionConflict``).
|
||||
|
||||
config.json can hold the CivitAI key, so the file is written owner-only.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
ABSENT_REVISION = "absent"
|
||||
|
||||
# One lock for the process: every ConfigStore writes the same file.
|
||||
_COMMIT_LOCK = threading.Lock()
|
||||
|
||||
|
||||
class ConfigUnreadable(Exception):
|
||||
"""config.json exists but is not a readable JSON object; it is left alone."""
|
||||
|
||||
|
||||
class RevisionConflict(Exception):
|
||||
"""config.json changed since the revision the caller saw."""
|
||||
|
||||
def __init__(self, current: str):
|
||||
super().__init__("config.json changed since it was read")
|
||||
self.current = current
|
||||
|
||||
|
||||
def revision_of(data: Optional[bytes]) -> str:
|
||||
"""The revision of a file's bytes; ``None`` means the file is absent."""
|
||||
if data is None:
|
||||
return ABSENT_REVISION
|
||||
return "sha256:" + hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _default_path() -> str:
|
||||
from .config import PromptManagerConfig
|
||||
|
||||
return PromptManagerConfig.get_config_path()
|
||||
|
||||
|
||||
def _write_atomically(path: str, payload: bytes) -> None:
|
||||
"""Replace ``path`` with ``payload``: temp file beside it, fsync, replace.
|
||||
|
||||
The descriptor is closed before ``os.replace`` (Windows refuses to replace
|
||||
over an open handle), and on any failure the temp file is closed and
|
||||
removed, so a failed write leaves the old file and no stray temp file.
|
||||
"""
|
||||
parent = os.path.dirname(path) or "."
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
fd, temp_path = tempfile.mkstemp(dir=parent, prefix=".config-", suffix=".tmp")
|
||||
try:
|
||||
try:
|
||||
handle = os.fdopen(fd, "wb")
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
raise
|
||||
with handle: # closed before the replace, also when a write fails
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
if os.name == "posix":
|
||||
os.chmod(temp_path, 0o600)
|
||||
os.replace(temp_path, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
class ConfigStore:
|
||||
"""Serialized, atomic reads and commits of config.json."""
|
||||
|
||||
def __init__(self, path_getter: Optional[Callable[[], str]] = None):
|
||||
self._path_getter = path_getter or _default_path
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""The config file this store reads and writes (resolved per call)."""
|
||||
return self._path_getter()
|
||||
|
||||
def read(self) -> Tuple[dict, str]:
|
||||
"""``(config, revision)``; a missing file is ``({}, "absent")``.
|
||||
|
||||
Raises ConfigUnreadable when the file cannot be read, is not JSON, or
|
||||
is not a JSON object.
|
||||
"""
|
||||
return self._read(self.path)
|
||||
|
||||
@staticmethod
|
||||
def _read(path: str) -> Tuple[dict, str]:
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
raw = handle.read()
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
# No file there (a parent that is a file cannot hold one either);
|
||||
# a commit then fails on the write, as config-write-failed.
|
||||
return {}, ABSENT_REVISION
|
||||
except OSError as exc:
|
||||
raise ConfigUnreadable(f"config.json cannot be read: {exc}") from exc
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError) as exc:
|
||||
raise ConfigUnreadable("config.json is not valid JSON") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ConfigUnreadable("config.json is not a JSON object")
|
||||
return data, revision_of(raw)
|
||||
|
||||
def commit(
|
||||
self,
|
||||
mutate: Callable[[dict], dict],
|
||||
*,
|
||||
expected_revision: Optional[str],
|
||||
) -> Tuple[dict, str]:
|
||||
"""Apply ``mutate`` to the current config and write the result.
|
||||
|
||||
``mutate`` receives a deep copy of the file's content and returns the
|
||||
whole new config (an object). ``expected_revision`` None merges into
|
||||
whatever is on disk (legacy partial updates); a revision refuses to
|
||||
write over different content. Returns ``(config, revision)`` as
|
||||
written. Raises ConfigUnreadable, RevisionConflict, TypeError (a
|
||||
non-object result) or OSError (the write failed; the old file stays).
|
||||
"""
|
||||
with _COMMIT_LOCK:
|
||||
path = self.path
|
||||
current, revision = self._read(path)
|
||||
if expected_revision is not None and expected_revision != revision:
|
||||
raise RevisionConflict(revision)
|
||||
candidate = mutate(copy.deepcopy(current))
|
||||
if not isinstance(candidate, dict):
|
||||
raise TypeError("A config commit must produce a JSON object")
|
||||
payload = json.dumps(candidate, indent=2).encode("utf-8")
|
||||
_write_atomically(path, payload)
|
||||
return candidate, revision_of(payload)
|
||||
|
||||
|
||||
_store = ConfigStore()
|
||||
|
||||
|
||||
def get_config_store() -> ConfigStore:
|
||||
"""The process's config store (on ``PromptManagerConfig.get_config_path``)."""
|
||||
return _store
|
||||
Reference in New Issue
Block a user