feat: implement hard reset functionality for database with enhanced safety checks and user options
This commit is contained in:
@@ -1102,6 +1102,8 @@ class Sqlite:
|
||||
4. Re-init
|
||||
"""
|
||||
try:
|
||||
deleted_files: list[str] = []
|
||||
renamed_files: list[dict[str, str]] = []
|
||||
# 1. Drain and close on loop thread
|
||||
fut = self._loop_thread.submit(self._drain_for_reset_async())
|
||||
await asyncio.wrap_future(fut)
|
||||
@@ -1123,6 +1125,7 @@ class Sqlite:
|
||||
try:
|
||||
p.unlink()
|
||||
deleted = True
|
||||
deleted_files.append(str(p))
|
||||
break
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
@@ -1131,17 +1134,21 @@ class Sqlite:
|
||||
|
||||
if not deleted:
|
||||
logger.warning(f"Failed to delete {p} after retries: {last_error}")
|
||||
# If main file fails, it's fatal. WAL/SHM might be tolerable or indicate locks.
|
||||
if ext == "":
|
||||
# Final Hail Mary: Try rename if delete fails (Windows hack)
|
||||
try:
|
||||
trash = p.with_name(f"{p.name}.trash_{uuid.uuid4().hex}")
|
||||
p.rename(trash)
|
||||
# Schedule trash deletion for later or ignore
|
||||
logger.info(f"Renamed locked DB file to {trash}")
|
||||
deleted = True
|
||||
except Exception as rename_err:
|
||||
return Result.Err("DELETE_FAILED", f"Could not delete {p}: {last_error} | Rename failed: {rename_err}")
|
||||
# Final Hail Mary: Try rename if delete fails (Windows hack).
|
||||
# We do this for main DB and WAL/SHM to guarantee the old files disappear.
|
||||
try:
|
||||
trash = p.with_name(f"{p.name}.trash_{uuid.uuid4().hex}")
|
||||
p.rename(trash)
|
||||
renamed_files.append({"from": str(p), "to": str(trash)})
|
||||
logger.info(f"Renamed locked DB file to {trash}")
|
||||
deleted = True
|
||||
except Exception as rename_err:
|
||||
# If main file fails, it's fatal. WAL/SHM failures are also fatal here
|
||||
# because the user explicitly requested a hard reset.
|
||||
return Result.Err(
|
||||
"DELETE_FAILED",
|
||||
f"Could not delete {p}: {last_error} | Rename failed: {rename_err}",
|
||||
)
|
||||
|
||||
# 4. Reset initialization state
|
||||
with self._lock:
|
||||
@@ -1160,8 +1167,8 @@ class Sqlite:
|
||||
indexes_res = await ensure_indexes_and_triggers(self)
|
||||
if not indexes_res.ok:
|
||||
return indexes_res
|
||||
|
||||
return Result.Ok(True)
|
||||
|
||||
return Result.Ok(True, deleted_files=deleted_files, renamed_files=renamed_files)
|
||||
except Exception as exc:
|
||||
self._resetting = False # Ensure unlock on error
|
||||
logger.error(f"DB Reset failed: {exc}")
|
||||
|
||||
@@ -54,10 +54,14 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None:
|
||||
@routes.post("/mjr/am/db/reset")
|
||||
async def db_reset(request: web.Request):
|
||||
"""
|
||||
Reset database and trigger rescan.
|
||||
Legacy DB reset endpoint (compat).
|
||||
|
||||
This performs a hard DB reset (delete + recreate SQLite files) and triggers a rescan.
|
||||
It is gated behind the same security checks as /mjr/am/index/reset.
|
||||
"""
|
||||
import asyncio
|
||||
from backend.config import WATCHER_PATHS
|
||||
from ..core.security import _require_operation_enabled, _require_write_access, _resolve_security_prefs
|
||||
|
||||
csrf = _csrf_error(request)
|
||||
if csrf:
|
||||
@@ -67,6 +71,15 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None:
|
||||
if error_result:
|
||||
return _json_response(error_result)
|
||||
|
||||
prefs = await _resolve_security_prefs(svc)
|
||||
op = _require_operation_enabled("reset_index", prefs=prefs)
|
||||
if not op.ok:
|
||||
return _json_response(op)
|
||||
|
||||
auth = _require_write_access(request)
|
||||
if not auth.ok:
|
||||
return _json_response(auth)
|
||||
|
||||
db = svc.get("db")
|
||||
index_service = svc.get("index")
|
||||
|
||||
@@ -85,4 +98,4 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None:
|
||||
asyncio.create_task(index_service.scan_directory(str(path), recursive=True, incremental=False))
|
||||
started_scans.append(str(path))
|
||||
|
||||
return _json_response(Result.Ok({"reset": True, "scans_triggered": started_scans}))
|
||||
return _json_response(Result.Ok({"reset": True, "scans_triggered": started_scans, "file_ops": (reset_res.meta or {})}))
|
||||
|
||||
+186
-21
@@ -755,6 +755,20 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
|
||||
fast = _bool_option(("fast",), True)
|
||||
background_metadata = _bool_option(("background_metadata", "backgroundMetadata"), True)
|
||||
|
||||
# Hard reset deletes and recreates the SQLite DB files (assets.sqlite, -wal, -shm).
|
||||
# This matches the "Reset Index Now" maintenance intent.
|
||||
hard_reset_db = _bool_option(
|
||||
(
|
||||
"hard_reset_db",
|
||||
"hardResetDb",
|
||||
"delete_db_files",
|
||||
"deleteDbFiles",
|
||||
"delete_db",
|
||||
"deleteDb",
|
||||
),
|
||||
False,
|
||||
)
|
||||
|
||||
target_roots: list[dict[str, str | None]] = []
|
||||
try:
|
||||
if scope in ("output", "outputs"):
|
||||
@@ -794,8 +808,97 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
|
||||
if not db:
|
||||
return _json_response(Result.Err("SERVICE_UNAVAILABLE", "Database service unavailable"))
|
||||
|
||||
# Best-effort coordination with concurrent scans.
|
||||
# Do NOT hold the scan lock while invoking scan_directory() (it also uses the same lock).
|
||||
scan_lock = None
|
||||
try:
|
||||
index_svc = svc.get("index")
|
||||
scan_lock = getattr(index_svc, "_scan_lock", None)
|
||||
except Exception:
|
||||
scan_lock = None
|
||||
|
||||
cache_prefixes: list[str] | None = None if scope == "all" else [entry["path"] for entry in target_roots]
|
||||
|
||||
# If the caller is effectively requesting a full wipe (the usual UI path),
|
||||
# treat it as a hard reset by default when scope=all.
|
||||
implied_full_clear = bool(clear_scan_journal and clear_metadata_cache and clear_asset_metadata and clear_assets_table)
|
||||
should_hard_reset_db = bool(scope == "all" and (hard_reset_db or implied_full_clear))
|
||||
|
||||
if should_hard_reset_db:
|
||||
# 1) Hard reset DB files (includes -wal/-shm) using adapter logic (Windows-safe).
|
||||
try:
|
||||
if scan_lock is not None and hasattr(scan_lock, "__aenter__"):
|
||||
async with scan_lock:
|
||||
reset_res = await db.areset()
|
||||
else:
|
||||
reset_res = await db.areset()
|
||||
except Exception as exc:
|
||||
logger.exception("Hard reset DB failed")
|
||||
return _json_response(Result.Err("RESET_FAILED", f"Hard reset failed: {exc}"))
|
||||
|
||||
if not reset_res.ok:
|
||||
return _json_response(reset_res)
|
||||
|
||||
cleared = {
|
||||
"scan_journal": None,
|
||||
"metadata_cache": None,
|
||||
"asset_metadata": None,
|
||||
"assets": None,
|
||||
"hard_reset_db": True,
|
||||
"file_ops": (reset_res.meta or {}),
|
||||
}
|
||||
|
||||
# 2) Optionally reindex after reset.
|
||||
scan_details: list[dict[str, Any]] = []
|
||||
totals = {key: 0 for key in ("scanned", "added", "updated", "skipped", "errors")}
|
||||
if reindex:
|
||||
if not svc.get("index"):
|
||||
return _json_response(Result.Err("SERVICE_UNAVAILABLE", "Index service unavailable"))
|
||||
for target in target_roots:
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
svc['index'].scan_directory(
|
||||
target["path"],
|
||||
True,
|
||||
incremental,
|
||||
target["source"],
|
||||
target.get("root_id"),
|
||||
fast,
|
||||
background_metadata,
|
||||
),
|
||||
timeout=TO_THREAD_TIMEOUT_S,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
return _json_response(Result.Err("TIMEOUT", "Index reset scan timed out"))
|
||||
except Exception as exc:
|
||||
logger.exception("Index reset scan failed")
|
||||
return _json_response(Result.Err("RESET_FAILED", f"Index reset scan failed: {exc}"))
|
||||
|
||||
if not result.ok:
|
||||
return _json_response(result)
|
||||
|
||||
stats = result.data or {}
|
||||
counts = {key: int(stats.get(key) or 0) for key in totals}
|
||||
for key in totals:
|
||||
totals[key] += counts[key]
|
||||
scan_details.append(
|
||||
{
|
||||
"source": target["source"],
|
||||
"root_id": target.get("root_id"),
|
||||
**counts,
|
||||
}
|
||||
)
|
||||
|
||||
scan_summary = {
|
||||
"scope": scope,
|
||||
"reindex": bool(reindex),
|
||||
"details": scan_details,
|
||||
**totals,
|
||||
}
|
||||
|
||||
# No need to VACUUM or rebuild FTS here: db.areset() reinitializes schema + indexes.
|
||||
return _json_response(Result.Ok({"cleared": cleared, "scan_summary": scan_summary, "rebuild_fts": None}))
|
||||
|
||||
async def _clear_table(table: str, prefixes: list[str] | None) -> Result[int]:
|
||||
total_deleted = 0
|
||||
if prefixes is None:
|
||||
@@ -818,27 +921,81 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
|
||||
total_deleted += int(res.data or 0)
|
||||
return Result.Ok(total_deleted)
|
||||
|
||||
async def _clear_assets(prefixes: list[str] | None) -> Result[int]:
|
||||
# Deleting from assets cascades to asset_metadata/scan_journal/metadata_cache.
|
||||
return await _clear_table("assets", prefixes)
|
||||
|
||||
async def _clear_asset_metadata(prefixes: list[str] | None) -> Result[int]:
|
||||
total_deleted = 0
|
||||
if prefixes is None:
|
||||
res = await db.aexecute("DELETE FROM asset_metadata")
|
||||
if not res.ok:
|
||||
return res
|
||||
return Result.Ok(int(res.data or 0))
|
||||
|
||||
for prefix in prefixes:
|
||||
try:
|
||||
normalized = str(Path(prefix).resolve(strict=False))
|
||||
except Exception:
|
||||
return Result.Err("INVALID_INPUT", f"Invalid path for metadata clearing: {prefix}")
|
||||
like = normalized.rstrip(os.path.sep) + os.path.sep + "%"
|
||||
res = await db.aexecute(
|
||||
"""
|
||||
DELETE FROM asset_metadata
|
||||
WHERE asset_id IN (
|
||||
SELECT id FROM assets
|
||||
WHERE filepath = ? OR filepath LIKE ?
|
||||
)
|
||||
""",
|
||||
(normalized, like),
|
||||
)
|
||||
if not res.ok:
|
||||
return res
|
||||
total_deleted += int(res.data or 0)
|
||||
return Result.Ok(total_deleted)
|
||||
|
||||
cleared = {"scan_journal": 0, "metadata_cache": 0, "asset_metadata": 0, "assets": 0}
|
||||
if clear_scan_journal:
|
||||
res = await _clear_table("scan_journal", cache_prefixes)
|
||||
if not res.ok:
|
||||
return _json_response(res)
|
||||
cleared["scan_journal"] = int(res.data or 0)
|
||||
if clear_metadata_cache:
|
||||
res = await _clear_table("metadata_cache", cache_prefixes)
|
||||
if not res.ok:
|
||||
return _json_response(res)
|
||||
cleared["metadata_cache"] = int(res.data or 0)
|
||||
if clear_asset_metadata:
|
||||
res = await db.aexecute("DELETE FROM asset_metadata")
|
||||
if not res.ok:
|
||||
return _json_response(res)
|
||||
cleared["asset_metadata"] = int(res.data or 0)
|
||||
if clear_assets_table:
|
||||
res = await db.aexecute("DELETE FROM assets")
|
||||
if not res.ok:
|
||||
return _json_response(res)
|
||||
cleared["assets"] = int(res.data or 0)
|
||||
|
||||
async def _do_clears() -> Result[dict]:
|
||||
# Make clears atomic and resistant to concurrent writers.
|
||||
async with db.atransaction(mode="immediate"):
|
||||
if clear_scan_journal:
|
||||
res = await _clear_table("scan_journal", cache_prefixes)
|
||||
if not res.ok:
|
||||
return res
|
||||
cleared["scan_journal"] = int(res.data or 0)
|
||||
if clear_metadata_cache:
|
||||
res = await _clear_table("metadata_cache", cache_prefixes)
|
||||
if not res.ok:
|
||||
return res
|
||||
cleared["metadata_cache"] = int(res.data or 0)
|
||||
|
||||
# IMPORTANT: scope-aware clears.
|
||||
# Previous implementation cleared the entire tables, even for output/custom scope.
|
||||
if clear_asset_metadata:
|
||||
res = await _clear_asset_metadata(cache_prefixes)
|
||||
if not res.ok:
|
||||
return res
|
||||
cleared["asset_metadata"] = int(res.data or 0)
|
||||
if clear_assets_table:
|
||||
res = await _clear_assets(cache_prefixes)
|
||||
if not res.ok:
|
||||
return res
|
||||
cleared["assets"] = int(res.data or 0)
|
||||
return Result.Ok(cleared)
|
||||
|
||||
try:
|
||||
if scan_lock is not None and hasattr(scan_lock, "__aenter__"):
|
||||
async with scan_lock:
|
||||
res = await _do_clears()
|
||||
else:
|
||||
res = await _do_clears()
|
||||
except Exception as exc:
|
||||
logger.exception("Index reset clear phase failed")
|
||||
return _json_response(Result.Err("RESET_FAILED", f"Index reset failed during clear phase: {exc}"))
|
||||
|
||||
if not res.ok:
|
||||
return _json_response(res)
|
||||
|
||||
# FULL RESET CLEANUP: VACUUM and Physical Files
|
||||
if scope == "all" and (clear_scan_journal or clear_metadata_cache):
|
||||
@@ -858,6 +1015,9 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
|
||||
continue
|
||||
if item == COLLECTIONS_DIR_PATH:
|
||||
continue
|
||||
# Preserve user configuration/state stored in the index directory.
|
||||
if item.name == "custom_roots.json":
|
||||
continue
|
||||
|
||||
try:
|
||||
if item.is_dir():
|
||||
@@ -917,7 +1077,12 @@ def register_scan_routes(routes: web.RouteTableDef) -> None:
|
||||
|
||||
rebuild_status = None
|
||||
if rebuild_fts_flag:
|
||||
rebuild_result = await rebuild_fts(db)
|
||||
try:
|
||||
async with db.atransaction(mode="immediate"):
|
||||
rebuild_result = await rebuild_fts(db)
|
||||
except Exception as exc:
|
||||
logger.exception("FTS rebuild failed during index reset")
|
||||
return _json_response(Result.Err("DB_ERROR", f"FTS rebuild failed: {exc}"))
|
||||
if not rebuild_result.ok:
|
||||
return _json_response(rebuild_result)
|
||||
rebuild_status = True
|
||||
|
||||
@@ -458,6 +458,17 @@ export async function resetIndex(options = {}) {
|
||||
const body = {
|
||||
scope,
|
||||
reindex: _bool(options.reindex, true),
|
||||
// When scope=all, the backend defaults to a hard DB reset unless explicitly disabled.
|
||||
hard_reset_db: _bool(
|
||||
options.hardResetDb ??
|
||||
options.hard_reset_db ??
|
||||
options.deleteDbFiles ??
|
||||
options.delete_db_files ??
|
||||
options.deleteDb ??
|
||||
options.delete_db ??
|
||||
undefined,
|
||||
scope === "all"
|
||||
),
|
||||
clear_scan_journal: _bool(options.clearScanJournal ?? options.clear_scan_journal, true),
|
||||
clear_metadata_cache: _bool(options.clearMetadataCache ?? options.clear_metadata_cache, true),
|
||||
clear_asset_metadata: _bool(options.clearAssetMetadata ?? options.clear_asset_metadata, true),
|
||||
|
||||
+2
-1
@@ -16,7 +16,8 @@ export const ENDPOINTS = {
|
||||
// Index & Search
|
||||
SCAN: "/mjr/am/scan",
|
||||
INDEX_FILES: "/mjr/am/index-files",
|
||||
INDEX_RESET: "/mjr/am/db/reset",
|
||||
INDEX_RESET: "/mjr/am/index/reset",
|
||||
DB_RESET: "/mjr/am/db/reset",
|
||||
SEARCH: "/mjr/am/search",
|
||||
LIST: "/mjr/am/list",
|
||||
ROOTS: "/mjr/am/roots",
|
||||
|
||||
+16
-4
@@ -787,7 +787,7 @@ export const registerMajoorSettings = (app, onApplied) => {
|
||||
id: `${SETTINGS_PREFIX}.Maintenance.ResetIndexRun`,
|
||||
category: cat("Maintenance", "Reset Index Now"),
|
||||
name: "⚠️ Reset Index Now",
|
||||
tooltip: "Delete database & rescan (requires 'Allow Reset Index' enabled above)",
|
||||
tooltip: "Delete assets.sqlite (+ -wal/-shm) & rescan (requires 'Allow Reset Index')",
|
||||
type: "boolean",
|
||||
defaultValue: false,
|
||||
onChange: async (value) => {
|
||||
@@ -830,17 +830,29 @@ export const registerMajoorSettings = (app, onApplied) => {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!confirm("Are you SURE you want to delete the index database? This cannot be undone. A full rescan will start.")) {
|
||||
if (!confirm("Are you SURE you want to DELETE the index database (assets.sqlite + WAL/SHM)? This cannot be undone. A full rescan will start.")) {
|
||||
const s = app.ui.settings.settingAry.find(s => s.id === `${SETTINGS_PREFIX}.Maintenance.ResetIndexRun`);
|
||||
if (s) s.value = false;
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[Majoor] Requesting Index Reset...");
|
||||
comfyToast({ severity: 'info', summary: 'Resetting...', detail: 'Deleting database and restarting scan.' }, 'info');
|
||||
comfyToast({ severity: 'info', summary: 'Resetting...', detail: 'Deleting DB files and restarting scan.' }, 'info');
|
||||
|
||||
try {
|
||||
const res = await resetIndex();
|
||||
const res = await resetIndex({
|
||||
scope: "all",
|
||||
reindex: true,
|
||||
hard_reset_db: true,
|
||||
clear_scan_journal: true,
|
||||
clear_metadata_cache: true,
|
||||
clear_asset_metadata: true,
|
||||
clear_assets: true,
|
||||
rebuild_fts: true,
|
||||
incremental: false,
|
||||
fast: true,
|
||||
background_metadata: true,
|
||||
});
|
||||
if (res.ok) {
|
||||
console.log("[Majoor] Index Reset SUCCESS:", res.data);
|
||||
comfyToast({
|
||||
|
||||
@@ -152,6 +152,7 @@ export function createStatusIndicator() {
|
||||
const res = await resetIndex({
|
||||
scope: "all",
|
||||
reindex: true,
|
||||
hard_reset_db: true,
|
||||
clear_scan_journal: true,
|
||||
clear_metadata_cache: true,
|
||||
clear_asset_metadata: true,
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
from backend.adapters.db.sqlite import Sqlite
|
||||
from backend.adapters.db.schema import ensure_tables_exist, ensure_indexes_and_triggers
|
||||
from backend.routes.handlers.scan import register_scan_routes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_index_hard_reset_deletes_and_recreates_db(monkeypatch, tmp_path: Path):
|
||||
# Reset is gated behind an explicit opt-in for safety.
|
||||
monkeypatch.setenv("MAJOOR_ALLOW_RESET_INDEX", "1")
|
||||
|
||||
db_path = tmp_path / "assets.sqlite"
|
||||
db = Sqlite(str(db_path))
|
||||
|
||||
# Initialize schema + create some data so the file definitely exists.
|
||||
schema_res = await ensure_tables_exist(db)
|
||||
assert schema_res.ok, schema_res.error
|
||||
idx_res = await ensure_indexes_and_triggers(db)
|
||||
assert idx_res.ok, idx_res.error
|
||||
|
||||
await db.aexecute("INSERT INTO assets(filename, filepath, kind, ext, size, mtime) VALUES(?,?,?,?,?,?)", (
|
||||
"a.png",
|
||||
str(tmp_path / "a.png"),
|
||||
"image",
|
||||
"png",
|
||||
1,
|
||||
1,
|
||||
))
|
||||
|
||||
assert db_path.exists()
|
||||
before_mtime = db_path.stat().st_mtime
|
||||
|
||||
# Ensure WAL/SHM can exist on some platforms (best-effort; not strictly required).
|
||||
wal_path = Path(str(db_path) + "-wal")
|
||||
shm_path = Path(str(db_path) + "-shm")
|
||||
|
||||
# Monkeypatch scan handler to run without ComfyUI folder_paths.
|
||||
import backend.routes.handlers.scan as scan_mod
|
||||
|
||||
class _FP:
|
||||
@staticmethod
|
||||
def get_input_directory():
|
||||
p = tmp_path / "input"
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
return str(p)
|
||||
|
||||
monkeypatch.setattr(scan_mod, "folder_paths", _FP, raising=False)
|
||||
monkeypatch.setattr(scan_mod, "OUTPUT_ROOT", str(tmp_path / "output"), raising=False)
|
||||
|
||||
async def _mock_require_services():
|
||||
# No index service needed because reindex=False.
|
||||
return ({"db": db}, None)
|
||||
|
||||
monkeypatch.setattr(scan_mod, "_require_services", _mock_require_services)
|
||||
|
||||
routes = web.RouteTableDef()
|
||||
register_scan_routes(routes)
|
||||
app = web.Application()
|
||||
app.add_routes(routes)
|
||||
|
||||
client = TestClient(TestServer(app))
|
||||
await client.start_server()
|
||||
try:
|
||||
time.sleep(1.05) # mtime resolution guard (Windows)
|
||||
resp = await client.post(
|
||||
"/mjr/am/index/reset",
|
||||
data=json.dumps(
|
||||
{
|
||||
"scope": "all",
|
||||
"reindex": False,
|
||||
"hard_reset_db": True,
|
||||
"rebuild_fts": False,
|
||||
}
|
||||
),
|
||||
headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
payload = await resp.json()
|
||||
assert payload["ok"] is True, payload
|
||||
|
||||
assert db_path.exists()
|
||||
after_mtime = db_path.stat().st_mtime
|
||||
assert after_mtime >= before_mtime
|
||||
|
||||
# Data should be gone after hard reset.
|
||||
q = await db.aquery("SELECT COUNT(*) as c FROM assets")
|
||||
assert q.ok
|
||||
count = int((q.data or [{}])[0].get("c") or 0)
|
||||
assert count == 0
|
||||
|
||||
# WAL/SHM are adapter-managed; they may or may not exist immediately.
|
||||
# If they do exist, they should be regular files.
|
||||
if wal_path.exists():
|
||||
assert wal_path.is_file()
|
||||
if shm_path.exists():
|
||||
assert shm_path.is_file()
|
||||
finally:
|
||||
await client.close()
|
||||
await db.aclose()
|
||||
@@ -0,0 +1,132 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
|
||||
from backend.adapters.db.sqlite import Sqlite
|
||||
from backend.routes.handlers.scan import register_scan_routes
|
||||
|
||||
|
||||
async def _init_schema(db: Sqlite) -> None:
|
||||
await db.aexecutescript(
|
||||
"""
|
||||
PRAGMA foreign_keys=ON;
|
||||
|
||||
CREATE TABLE assets (
|
||||
id INTEGER PRIMARY KEY,
|
||||
filepath TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE asset_metadata (
|
||||
asset_id INTEGER PRIMARY KEY,
|
||||
metadata_raw TEXT DEFAULT '{}',
|
||||
FOREIGN KEY (asset_id) REFERENCES assets(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE scan_journal (
|
||||
filepath TEXT PRIMARY KEY,
|
||||
state_hash TEXT,
|
||||
FOREIGN KEY (filepath) REFERENCES assets(filepath) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE metadata_cache (
|
||||
filepath TEXT PRIMARY KEY,
|
||||
state_hash TEXT,
|
||||
metadata_raw TEXT DEFAULT '{}',
|
||||
FOREIGN KEY (filepath) REFERENCES assets(filepath) ON DELETE CASCADE
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reset_index_output_scope_does_not_wipe_other_roots(monkeypatch, tmp_path: Path):
|
||||
# Reset is gated behind an explicit opt-in for safety.
|
||||
monkeypatch.setenv("MAJOOR_ALLOW_RESET_INDEX", "1")
|
||||
|
||||
db = Sqlite(str(tmp_path / "test.db"))
|
||||
await _init_schema(db)
|
||||
|
||||
out_dir = tmp_path / "out"
|
||||
in_dir = tmp_path / "in"
|
||||
custom_dir = tmp_path / "custom"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
in_dir.mkdir(parents=True, exist_ok=True)
|
||||
custom_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fp_out = str(out_dir / "a.png")
|
||||
fp_in = str(in_dir / "b.png")
|
||||
fp_custom = str(custom_dir / "c.png")
|
||||
|
||||
await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (1, fp_out))
|
||||
await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (2, fp_in))
|
||||
await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (3, fp_custom))
|
||||
|
||||
await db.aexecute("INSERT INTO asset_metadata(asset_id, metadata_raw) VALUES (?, ?)", (1, "{\"x\": 1}"))
|
||||
await db.aexecute("INSERT INTO asset_metadata(asset_id, metadata_raw) VALUES (?, ?)", (2, "{\"x\": 2}"))
|
||||
await db.aexecute("INSERT INTO asset_metadata(asset_id, metadata_raw) VALUES (?, ?)", (3, "{\"x\": 3}"))
|
||||
|
||||
await db.aexecute("INSERT INTO scan_journal(filepath, state_hash) VALUES (?, ?)", (fp_out, "o"))
|
||||
await db.aexecute("INSERT INTO scan_journal(filepath, state_hash) VALUES (?, ?)", (fp_in, "i"))
|
||||
await db.aexecute("INSERT INTO scan_journal(filepath, state_hash) VALUES (?, ?)", (fp_custom, "c"))
|
||||
|
||||
await db.aexecute("INSERT INTO metadata_cache(filepath, state_hash, metadata_raw) VALUES (?, ?, ?)", (fp_out, "o", "{}"))
|
||||
await db.aexecute("INSERT INTO metadata_cache(filepath, state_hash, metadata_raw) VALUES (?, ?, ?)", (fp_in, "i", "{}"))
|
||||
await db.aexecute("INSERT INTO metadata_cache(filepath, state_hash, metadata_raw) VALUES (?, ?, ?)", (fp_custom, "c", "{}"))
|
||||
|
||||
import backend.routes.handlers.scan as scan_mod
|
||||
|
||||
async def _mock_require_services():
|
||||
return ({"db": db}, None)
|
||||
|
||||
monkeypatch.setattr(scan_mod, "_require_services", _mock_require_services)
|
||||
monkeypatch.setattr(scan_mod, "OUTPUT_ROOT", str(out_dir))
|
||||
|
||||
routes = web.RouteTableDef()
|
||||
register_scan_routes(routes)
|
||||
app = web.Application()
|
||||
app.add_routes(routes)
|
||||
|
||||
client = TestClient(TestServer(app))
|
||||
await client.start_server()
|
||||
try:
|
||||
resp = await client.post(
|
||||
"/mjr/am/index/reset",
|
||||
data=json.dumps(
|
||||
{
|
||||
"scope": "output",
|
||||
"reindex": False,
|
||||
"rebuild_fts": False,
|
||||
"clear_scan_journal": True,
|
||||
"clear_metadata_cache": True,
|
||||
"clear_asset_metadata": True,
|
||||
"clear_assets": True,
|
||||
}
|
||||
),
|
||||
headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
payload = await resp.json()
|
||||
assert payload["ok"] is True, payload
|
||||
|
||||
# Output entries should be removed; input/custom should remain.
|
||||
q_assets = await db.aquery("SELECT filepath FROM assets ORDER BY id")
|
||||
assert q_assets.ok
|
||||
remaining = [row["filepath"] for row in (q_assets.data or [])]
|
||||
assert remaining == [fp_in, fp_custom]
|
||||
|
||||
q_meta = await db.aquery("SELECT asset_id FROM asset_metadata ORDER BY asset_id")
|
||||
assert q_meta.ok
|
||||
assert [row["asset_id"] for row in (q_meta.data or [])] == [2, 3]
|
||||
|
||||
q_sj = await db.aquery("SELECT filepath FROM scan_journal ORDER BY filepath")
|
||||
assert q_sj.ok
|
||||
assert [row["filepath"] for row in (q_sj.data or [])] == sorted([fp_in, fp_custom])
|
||||
|
||||
q_mc = await db.aquery("SELECT filepath FROM metadata_cache ORDER BY filepath")
|
||||
assert q_mc.ok
|
||||
assert [row["filepath"] for row in (q_mc.data or [])] == sorted([fp_in, fp_custom])
|
||||
finally:
|
||||
await client.close()
|
||||
await db.aclose()
|
||||
Reference in New Issue
Block a user