feat: implement hard reset functionality for database with enhanced safety checks and user options

This commit is contained in:
Your Name
2026-02-04 00:58:02 +01:00
parent 7fda66a4d5
commit 85cd0507cf
9 changed files with 488 additions and 41 deletions
+20 -13
View File
@@ -1102,6 +1102,8 @@ class Sqlite:
4. Re-init
"""
try:
deleted_files: list[str] = []
renamed_files: list[dict[str, str]] = []
# 1. Drain and close on loop thread
fut = self._loop_thread.submit(self._drain_for_reset_async())
await asyncio.wrap_future(fut)
@@ -1123,6 +1125,7 @@ class Sqlite:
try:
p.unlink()
deleted = True
deleted_files.append(str(p))
break
except Exception as e:
last_error = e
@@ -1131,17 +1134,21 @@ class Sqlite:
if not deleted:
logger.warning(f"Failed to delete {p} after retries: {last_error}")
# If main file fails, it's fatal. WAL/SHM might be tolerable or indicate locks.
if ext == "":
# Final Hail Mary: Try rename if delete fails (Windows hack)
try:
trash = p.with_name(f"{p.name}.trash_{uuid.uuid4().hex}")
p.rename(trash)
# Schedule trash deletion for later or ignore
logger.info(f"Renamed locked DB file to {trash}")
deleted = True
except Exception as rename_err:
return Result.Err("DELETE_FAILED", f"Could not delete {p}: {last_error} | Rename failed: {rename_err}")
# Final Hail Mary: Try rename if delete fails (Windows hack).
# We do this for main DB and WAL/SHM to guarantee the old files disappear.
try:
trash = p.with_name(f"{p.name}.trash_{uuid.uuid4().hex}")
p.rename(trash)
renamed_files.append({"from": str(p), "to": str(trash)})
logger.info(f"Renamed locked DB file to {trash}")
deleted = True
except Exception as rename_err:
# If main file fails, it's fatal. WAL/SHM failures are also fatal here
# because the user explicitly requested a hard reset.
return Result.Err(
"DELETE_FAILED",
f"Could not delete {p}: {last_error} | Rename failed: {rename_err}",
)
# 4. Reset initialization state
with self._lock:
@@ -1160,8 +1167,8 @@ class Sqlite:
indexes_res = await ensure_indexes_and_triggers(self)
if not indexes_res.ok:
return indexes_res
return Result.Ok(True)
return Result.Ok(True, deleted_files=deleted_files, renamed_files=renamed_files)
except Exception as exc:
self._resetting = False # Ensure unlock on error
logger.error(f"DB Reset failed: {exc}")
+15 -2
View File
@@ -54,10 +54,14 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None:
@routes.post("/mjr/am/db/reset")
async def db_reset(request: web.Request):
"""
Reset database and trigger rescan.
Legacy DB reset endpoint (compat).
This performs a hard DB reset (delete + recreate SQLite files) and triggers a rescan.
It is gated behind the same security checks as /mjr/am/index/reset.
"""
import asyncio
from backend.config import WATCHER_PATHS
from ..core.security import _require_operation_enabled, _require_write_access, _resolve_security_prefs
csrf = _csrf_error(request)
if csrf:
@@ -67,6 +71,15 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None:
if error_result:
return _json_response(error_result)
prefs = await _resolve_security_prefs(svc)
op = _require_operation_enabled("reset_index", prefs=prefs)
if not op.ok:
return _json_response(op)
auth = _require_write_access(request)
if not auth.ok:
return _json_response(auth)
db = svc.get("db")
index_service = svc.get("index")
@@ -85,4 +98,4 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None:
asyncio.create_task(index_service.scan_directory(str(path), recursive=True, incremental=False))
started_scans.append(str(path))
return _json_response(Result.Ok({"reset": True, "scans_triggered": started_scans}))
return _json_response(Result.Ok({"reset": True, "scans_triggered": started_scans, "file_ops": (reset_res.meta or {})}))
+186 -21
View File
@@ -755,6 +755,20 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
fast = _bool_option(("fast",), True)
background_metadata = _bool_option(("background_metadata", "backgroundMetadata"), True)
# Hard reset deletes and recreates the SQLite DB files (assets.sqlite, -wal, -shm).
# This matches the "Reset Index Now" maintenance intent.
hard_reset_db = _bool_option(
(
"hard_reset_db",
"hardResetDb",
"delete_db_files",
"deleteDbFiles",
"delete_db",
"deleteDb",
),
False,
)
target_roots: list[dict[str, str | None]] = []
try:
if scope in ("output", "outputs"):
@@ -794,8 +808,97 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
if not db:
return _json_response(Result.Err("SERVICE_UNAVAILABLE", "Database service unavailable"))
# Best-effort coordination with concurrent scans.
# Do NOT hold the scan lock while invoking scan_directory() (it also uses the same lock).
scan_lock = None
try:
index_svc = svc.get("index")
scan_lock = getattr(index_svc, "_scan_lock", None)
except Exception:
scan_lock = None
cache_prefixes: list[str] | None = None if scope == "all" else [entry["path"] for entry in target_roots]
# If the caller is effectively requesting a full wipe (the usual UI path),
# treat it as a hard reset by default when scope=all.
implied_full_clear = bool(clear_scan_journal and clear_metadata_cache and clear_asset_metadata and clear_assets_table)
should_hard_reset_db = bool(scope == "all" and (hard_reset_db or implied_full_clear))
if should_hard_reset_db:
# 1) Hard reset DB files (includes -wal/-shm) using adapter logic (Windows-safe).
try:
if scan_lock is not None and hasattr(scan_lock, "__aenter__"):
async with scan_lock:
reset_res = await db.areset()
else:
reset_res = await db.areset()
except Exception as exc:
logger.exception("Hard reset DB failed")
return _json_response(Result.Err("RESET_FAILED", f"Hard reset failed: {exc}"))
if not reset_res.ok:
return _json_response(reset_res)
cleared = {
"scan_journal": None,
"metadata_cache": None,
"asset_metadata": None,
"assets": None,
"hard_reset_db": True,
"file_ops": (reset_res.meta or {}),
}
# 2) Optionally reindex after reset.
scan_details: list[dict[str, Any]] = []
totals = {key: 0 for key in ("scanned", "added", "updated", "skipped", "errors")}
if reindex:
if not svc.get("index"):
return _json_response(Result.Err("SERVICE_UNAVAILABLE", "Index service unavailable"))
for target in target_roots:
try:
result = await asyncio.wait_for(
svc['index'].scan_directory(
target["path"],
True,
incremental,
target["source"],
target.get("root_id"),
fast,
background_metadata,
),
timeout=TO_THREAD_TIMEOUT_S,
)
except asyncio.TimeoutError:
return _json_response(Result.Err("TIMEOUT", "Index reset scan timed out"))
except Exception as exc:
logger.exception("Index reset scan failed")
return _json_response(Result.Err("RESET_FAILED", f"Index reset scan failed: {exc}"))
if not result.ok:
return _json_response(result)
stats = result.data or {}
counts = {key: int(stats.get(key) or 0) for key in totals}
for key in totals:
totals[key] += counts[key]
scan_details.append(
{
"source": target["source"],
"root_id": target.get("root_id"),
**counts,
}
)
scan_summary = {
"scope": scope,
"reindex": bool(reindex),
"details": scan_details,
**totals,
}
# No need to VACUUM or rebuild FTS here: db.areset() reinitializes schema + indexes.
return _json_response(Result.Ok({"cleared": cleared, "scan_summary": scan_summary, "rebuild_fts": None}))
async def _clear_table(table: str, prefixes: list[str] | None) -> Result[int]:
total_deleted = 0
if prefixes is None:
@@ -818,27 +921,81 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
total_deleted += int(res.data or 0)
return Result.Ok(total_deleted)
async def _clear_assets(prefixes: list[str] | None) -> Result[int]:
# Deleting from assets cascades to asset_metadata/scan_journal/metadata_cache.
return await _clear_table("assets", prefixes)
async def _clear_asset_metadata(prefixes: list[str] | None) -> Result[int]:
total_deleted = 0
if prefixes is None:
res = await db.aexecute("DELETE FROM asset_metadata")
if not res.ok:
return res
return Result.Ok(int(res.data or 0))
for prefix in prefixes:
try:
normalized = str(Path(prefix).resolve(strict=False))
except Exception:
return Result.Err("INVALID_INPUT", f"Invalid path for metadata clearing: {prefix}")
like = normalized.rstrip(os.path.sep) + os.path.sep + "%"
res = await db.aexecute(
"""
DELETE FROM asset_metadata
WHERE asset_id IN (
SELECT id FROM assets
WHERE filepath = ? OR filepath LIKE ?
)
""",
(normalized, like),
)
if not res.ok:
return res
total_deleted += int(res.data or 0)
return Result.Ok(total_deleted)
cleared = {"scan_journal": 0, "metadata_cache": 0, "asset_metadata": 0, "assets": 0}
if clear_scan_journal:
res = await _clear_table("scan_journal", cache_prefixes)
if not res.ok:
return _json_response(res)
cleared["scan_journal"] = int(res.data or 0)
if clear_metadata_cache:
res = await _clear_table("metadata_cache", cache_prefixes)
if not res.ok:
return _json_response(res)
cleared["metadata_cache"] = int(res.data or 0)
if clear_asset_metadata:
res = await db.aexecute("DELETE FROM asset_metadata")
if not res.ok:
return _json_response(res)
cleared["asset_metadata"] = int(res.data or 0)
if clear_assets_table:
res = await db.aexecute("DELETE FROM assets")
if not res.ok:
return _json_response(res)
cleared["assets"] = int(res.data or 0)
async def _do_clears() -> Result[dict]:
# Make clears atomic and resistant to concurrent writers.
async with db.atransaction(mode="immediate"):
if clear_scan_journal:
res = await _clear_table("scan_journal", cache_prefixes)
if not res.ok:
return res
cleared["scan_journal"] = int(res.data or 0)
if clear_metadata_cache:
res = await _clear_table("metadata_cache", cache_prefixes)
if not res.ok:
return res
cleared["metadata_cache"] = int(res.data or 0)
# IMPORTANT: scope-aware clears.
# Previous implementation cleared the entire tables, even for output/custom scope.
if clear_asset_metadata:
res = await _clear_asset_metadata(cache_prefixes)
if not res.ok:
return res
cleared["asset_metadata"] = int(res.data or 0)
if clear_assets_table:
res = await _clear_assets(cache_prefixes)
if not res.ok:
return res
cleared["assets"] = int(res.data or 0)
return Result.Ok(cleared)
try:
if scan_lock is not None and hasattr(scan_lock, "__aenter__"):
async with scan_lock:
res = await _do_clears()
else:
res = await _do_clears()
except Exception as exc:
logger.exception("Index reset clear phase failed")
return _json_response(Result.Err("RESET_FAILED", f"Index reset failed during clear phase: {exc}"))
if not res.ok:
return _json_response(res)
# FULL RESET CLEANUP: VACUUM and Physical Files
if scope == "all" and (clear_scan_journal or clear_metadata_cache):
@@ -858,6 +1015,9 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
continue
if item == COLLECTIONS_DIR_PATH:
continue
# Preserve user configuration/state stored in the index directory.
if item.name == "custom_roots.json":
continue
try:
if item.is_dir():
@@ -917,7 +1077,12 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
rebuild_status = None
if rebuild_fts_flag:
rebuild_result = await rebuild_fts(db)
try:
async with db.atransaction(mode="immediate"):
rebuild_result = await rebuild_fts(db)
except Exception as exc:
logger.exception("FTS rebuild failed during index reset")
return _json_response(Result.Err("DB_ERROR", f"FTS rebuild failed: {exc}"))
if not rebuild_result.ok:
return _json_response(rebuild_result)
rebuild_status = True
+11
View File
@@ -458,6 +458,17 @@ export async function resetIndex(options = {}) {
const body = {
scope,
reindex: _bool(options.reindex, true),
// When scope=all, the backend defaults to a hard DB reset unless explicitly disabled.
hard_reset_db: _bool(
options.hardResetDb ??
options.hard_reset_db ??
options.deleteDbFiles ??
options.delete_db_files ??
options.deleteDb ??
options.delete_db ??
undefined,
scope === "all"
),
clear_scan_journal: _bool(options.clearScanJournal ?? options.clear_scan_journal, true),
clear_metadata_cache: _bool(options.clearMetadataCache ?? options.clear_metadata_cache, true),
clear_asset_metadata: _bool(options.clearAssetMetadata ?? options.clear_asset_metadata, true),
+2 -1
View File
@@ -16,7 +16,8 @@ export const ENDPOINTS = {
// Index & Search
SCAN: "/mjr/am/scan",
INDEX_FILES: "/mjr/am/index-files",
INDEX_RESET: "/mjr/am/db/reset",
INDEX_RESET: "/mjr/am/index/reset",
DB_RESET: "/mjr/am/db/reset",
SEARCH: "/mjr/am/search",
LIST: "/mjr/am/list",
ROOTS: "/mjr/am/roots",
+16 -4
View File
@@ -787,7 +787,7 @@ export const registerMajoorSettings = (app, onApplied) => {
id: `${SETTINGS_PREFIX}.Maintenance.ResetIndexRun`,
category: cat("Maintenance", "Reset Index Now"),
name: "⚠️ Reset Index Now",
tooltip: "Delete database & rescan (requires 'Allow Reset Index' enabled above)",
tooltip: "Delete assets.sqlite (+ -wal/-shm) & rescan (requires 'Allow Reset Index')",
type: "boolean",
defaultValue: false,
onChange: async (value) => {
@@ -830,17 +830,29 @@ export const registerMajoorSettings = (app, onApplied) => {
return;
}
if (!confirm("Are you SURE you want to delete the index database? This cannot be undone. A full rescan will start.")) {
if (!confirm("Are you SURE you want to DELETE the index database (assets.sqlite + WAL/SHM)? This cannot be undone. A full rescan will start.")) {
const s = app.ui.settings.settingAry.find(s => s.id === `${SETTINGS_PREFIX}.Maintenance.ResetIndexRun`);
if (s) s.value = false;
return;
}
console.log("[Majoor] Requesting Index Reset...");
comfyToast({ severity: 'info', summary: 'Resetting...', detail: 'Deleting database and restarting scan.' }, 'info');
comfyToast({ severity: 'info', summary: 'Resetting...', detail: 'Deleting DB files and restarting scan.' }, 'info');
try {
const res = await resetIndex();
const res = await resetIndex({
scope: "all",
reindex: true,
hard_reset_db: true,
clear_scan_journal: true,
clear_metadata_cache: true,
clear_asset_metadata: true,
clear_assets: true,
rebuild_fts: true,
incremental: false,
fast: true,
background_metadata: true,
});
if (res.ok) {
console.log("[Majoor] Index Reset SUCCESS:", res.data);
comfyToast({
+1
View File
@@ -152,6 +152,7 @@ export function createStatusIndicator() {
const res = await resetIndex({
scope: "all",
reindex: true,
hard_reset_db: true,
clear_scan_journal: true,
clear_metadata_cache: true,
clear_asset_metadata: true,
@@ -0,0 +1,105 @@
import json
import time
from pathlib import Path
import pytest
from aiohttp import web
from aiohttp.test_utils import TestClient, TestServer
from backend.adapters.db.sqlite import Sqlite
from backend.adapters.db.schema import ensure_tables_exist, ensure_indexes_and_triggers
from backend.routes.handlers.scan import register_scan_routes
@pytest.mark.asyncio
async def test_reset_index_hard_reset_deletes_and_recreates_db(monkeypatch, tmp_path: Path):
# Reset is gated behind an explicit opt-in for safety.
monkeypatch.setenv("MAJOOR_ALLOW_RESET_INDEX", "1")
db_path = tmp_path / "assets.sqlite"
db = Sqlite(str(db_path))
# Initialize schema + create some data so the file definitely exists.
schema_res = await ensure_tables_exist(db)
assert schema_res.ok, schema_res.error
idx_res = await ensure_indexes_and_triggers(db)
assert idx_res.ok, idx_res.error
await db.aexecute("INSERT INTO assets(filename, filepath, kind, ext, size, mtime) VALUES(?,?,?,?,?,?)", (
"a.png",
str(tmp_path / "a.png"),
"image",
"png",
1,
1,
))
assert db_path.exists()
before_mtime = db_path.stat().st_mtime
# Ensure WAL/SHM can exist on some platforms (best-effort; not strictly required).
wal_path = Path(str(db_path) + "-wal")
shm_path = Path(str(db_path) + "-shm")
# Monkeypatch scan handler to run without ComfyUI folder_paths.
import backend.routes.handlers.scan as scan_mod
class _FP:
@staticmethod
def get_input_directory():
p = tmp_path / "input"
p.mkdir(parents=True, exist_ok=True)
return str(p)
monkeypatch.setattr(scan_mod, "folder_paths", _FP, raising=False)
monkeypatch.setattr(scan_mod, "OUTPUT_ROOT", str(tmp_path / "output"), raising=False)
async def _mock_require_services():
# No index service needed because reindex=False.
return ({"db": db}, None)
monkeypatch.setattr(scan_mod, "_require_services", _mock_require_services)
routes = web.RouteTableDef()
register_scan_routes(routes)
app = web.Application()
app.add_routes(routes)
client = TestClient(TestServer(app))
await client.start_server()
try:
time.sleep(1.05) # mtime resolution guard (Windows)
resp = await client.post(
"/mjr/am/index/reset",
data=json.dumps(
{
"scope": "all",
"reindex": False,
"hard_reset_db": True,
"rebuild_fts": False,
}
),
headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"},
)
payload = await resp.json()
assert payload["ok"] is True, payload
assert db_path.exists()
after_mtime = db_path.stat().st_mtime
assert after_mtime >= before_mtime
# Data should be gone after hard reset.
q = await db.aquery("SELECT COUNT(*) as c FROM assets")
assert q.ok
count = int((q.data or [{}])[0].get("c") or 0)
assert count == 0
# WAL/SHM are adapter-managed; they may or may not exist immediately.
# If they do exist, they should be regular files.
if wal_path.exists():
assert wal_path.is_file()
if shm_path.exists():
assert shm_path.is_file()
finally:
await client.close()
await db.aclose()
+132
View File
@@ -0,0 +1,132 @@
import json
from pathlib import Path
import pytest
from aiohttp import web
from aiohttp.test_utils import TestClient, TestServer
from backend.adapters.db.sqlite import Sqlite
from backend.routes.handlers.scan import register_scan_routes
async def _init_schema(db: Sqlite) -> None:
await db.aexecutescript(
"""
PRAGMA foreign_keys=ON;
CREATE TABLE assets (
id INTEGER PRIMARY KEY,
filepath TEXT NOT NULL UNIQUE
);
CREATE TABLE asset_metadata (
asset_id INTEGER PRIMARY KEY,
metadata_raw TEXT DEFAULT '{}',
FOREIGN KEY (asset_id) REFERENCES assets(id) ON DELETE CASCADE
);
CREATE TABLE scan_journal (
filepath TEXT PRIMARY KEY,
state_hash TEXT,
FOREIGN KEY (filepath) REFERENCES assets(filepath) ON DELETE CASCADE
);
CREATE TABLE metadata_cache (
filepath TEXT PRIMARY KEY,
state_hash TEXT,
metadata_raw TEXT DEFAULT '{}',
FOREIGN KEY (filepath) REFERENCES assets(filepath) ON DELETE CASCADE
);
"""
)
@pytest.mark.asyncio
async def test_reset_index_output_scope_does_not_wipe_other_roots(monkeypatch, tmp_path: Path):
# Reset is gated behind an explicit opt-in for safety.
monkeypatch.setenv("MAJOOR_ALLOW_RESET_INDEX", "1")
db = Sqlite(str(tmp_path / "test.db"))
await _init_schema(db)
out_dir = tmp_path / "out"
in_dir = tmp_path / "in"
custom_dir = tmp_path / "custom"
out_dir.mkdir(parents=True, exist_ok=True)
in_dir.mkdir(parents=True, exist_ok=True)
custom_dir.mkdir(parents=True, exist_ok=True)
fp_out = str(out_dir / "a.png")
fp_in = str(in_dir / "b.png")
fp_custom = str(custom_dir / "c.png")
await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (1, fp_out))
await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (2, fp_in))
await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (3, fp_custom))
await db.aexecute("INSERT INTO asset_metadata(asset_id, metadata_raw) VALUES (?, ?)", (1, "{\"x\": 1}"))
await db.aexecute("INSERT INTO asset_metadata(asset_id, metadata_raw) VALUES (?, ?)", (2, "{\"x\": 2}"))
await db.aexecute("INSERT INTO asset_metadata(asset_id, metadata_raw) VALUES (?, ?)", (3, "{\"x\": 3}"))
await db.aexecute("INSERT INTO scan_journal(filepath, state_hash) VALUES (?, ?)", (fp_out, "o"))
await db.aexecute("INSERT INTO scan_journal(filepath, state_hash) VALUES (?, ?)", (fp_in, "i"))
await db.aexecute("INSERT INTO scan_journal(filepath, state_hash) VALUES (?, ?)", (fp_custom, "c"))
await db.aexecute("INSERT INTO metadata_cache(filepath, state_hash, metadata_raw) VALUES (?, ?, ?)", (fp_out, "o", "{}"))
await db.aexecute("INSERT INTO metadata_cache(filepath, state_hash, metadata_raw) VALUES (?, ?, ?)", (fp_in, "i", "{}"))
await db.aexecute("INSERT INTO metadata_cache(filepath, state_hash, metadata_raw) VALUES (?, ?, ?)", (fp_custom, "c", "{}"))
import backend.routes.handlers.scan as scan_mod
async def _mock_require_services():
return ({"db": db}, None)
monkeypatch.setattr(scan_mod, "_require_services", _mock_require_services)
monkeypatch.setattr(scan_mod, "OUTPUT_ROOT", str(out_dir))
routes = web.RouteTableDef()
register_scan_routes(routes)
app = web.Application()
app.add_routes(routes)
client = TestClient(TestServer(app))
await client.start_server()
try:
resp = await client.post(
"/mjr/am/index/reset",
data=json.dumps(
{
"scope": "output",
"reindex": False,
"rebuild_fts": False,
"clear_scan_journal": True,
"clear_metadata_cache": True,
"clear_asset_metadata": True,
"clear_assets": True,
}
),
headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"},
)
payload = await resp.json()
assert payload["ok"] is True, payload
# Output entries should be removed; input/custom should remain.
q_assets = await db.aquery("SELECT filepath FROM assets ORDER BY id")
assert q_assets.ok
remaining = [row["filepath"] for row in (q_assets.data or [])]
assert remaining == [fp_in, fp_custom]
q_meta = await db.aquery("SELECT asset_id FROM asset_metadata ORDER BY asset_id")
assert q_meta.ok
assert [row["asset_id"] for row in (q_meta.data or [])] == [2, 3]
q_sj = await db.aquery("SELECT filepath FROM scan_journal ORDER BY filepath")
assert q_sj.ok
assert [row["filepath"] for row in (q_sj.data or [])] == sorted([fp_in, fp_custom])
q_mc = await db.aquery("SELECT filepath FROM metadata_cache ORDER BY filepath")
assert q_mc.ok
assert [row["filepath"] for row in (q_mc.data or [])] == sorted([fp_in, fp_custom])
finally:
await client.close()
await db.aclose()