From 573f3aa1340ce99f4260c6e51a21b35a4e8beca1 Mon Sep 17 00:00:00 2001 From: Ewald <129623145+MajoorWaldi@users.noreply.github.com> Date: Fri, 6 Feb 2026 21:17:46 +0100 Subject: [PATCH] chore: save workspace changes --- .github/workflows/ci-python.yml | 28 + .github/workflows/nightly.yml | 34 + .github/workflows/release-on-tag.yml | 32 + AGENTS.md | 2 +- README.md | 28 + __init__.py | 1 - backend/adapters/db/schema.py | 8 +- backend/adapters/db/sqlite.py | 245 ++++--- backend/adapters/fs/__init__.py | 2 +- backend/config.py | 37 +- backend/custom_roots.py | 46 +- backend/deps.py | 81 ++- backend/features/health/service.py | 7 + backend/features/index/enricher.py | 102 ++- backend/features/index/metadata_helpers.py | 168 ++++- backend/features/index/scanner.py | 227 ++++-- backend/features/index/searcher.py | 28 +- backend/features/index/service.py | 1 - backend/features/index/updater.py | 66 +- backend/features/index/watcher.py | 661 ++++++++++++------ backend/features/index/watcher_scope.py | 107 +++ .../metadata/archived/native_extractors.py | 292 -------- backend/features/metadata/service.py | 33 + backend/routes/core/security.py | 7 +- backend/routes/core/services.py | 36 +- backend/routes/handlers/__init__.py | 2 + backend/routes/handlers/collections.py | 9 +- backend/routes/handlers/custom_roots.py | 83 ++- backend/routes/handlers/db_maintenance.py | 12 +- backend/routes/handlers/filesystem.py | 43 +- backend/routes/handlers/filesystem_new.py | 448 ------------ backend/routes/handlers/health.py | 12 + backend/routes/handlers/releases.py | 74 ++ backend/routes/handlers/scan.py | 372 +++++++++- backend/routes/registry.py | 8 + backend/settings.py | 9 + comfyui_extension.json | 7 + docs/INSTALLATION.md | 9 +- docs/SETTINGS_CONFIGURATION.md | 49 +- docs/USER_GUIDE.md | 10 +- js/api/client.js | 16 + js/api/endpoints.js | 5 + js/app/i18n.js | 22 +- js/app/settings.js | 72 +- js/components/Badges.js | 19 +- js/components/Card.js | 6 +- js/components/ContextMenu.js | 491 ------------- js/components/Viewer.js | 21 +- js/entry.js | 474 ++++--------- js/features/contextmenu/GridContextMenu.js | 98 ++- js/features/dnd/out/DragOut.js | 25 +- js/features/grid/GridKeyboard.js | 94 ++- js/features/grid/GridView.js | 235 ++++++- js/features/grid/VirtualGrid.js | 21 +- js/features/panel/AssetsManagerPanel.js | 44 +- .../panel/controllers/contextController.js | 40 +- .../controllers/customRootsController.js | 6 +- .../panel/controllers/scopeController.js | 5 +- .../panel/controllers/sidebarController.js | 132 +++- js/features/status/StatusDot.js | 106 ++- js/features/viewer/genInfo.js | 102 +++ js/utils/dom.js | 26 + js/utils/extractOutputFiles.js | 104 ++- js/utils/fastUpload.js | 40 +- pyproject.toml | 2 +- requirements.txt | 5 +- shared/scan_throttle.py | 31 +- tests/README.md | 2 - tests/config/test_comfy_output.py | 4 +- tests/conftest.py | 4 +- tests/core/test_index.py | 4 +- tests/core/test_integration.py | 6 +- tests/features/test_fast_scan.py | 4 +- tests/features/test_open_in_folder.py | 73 ++ tests/features/test_upload_input.py | 42 ++ tests/features/test_watcher_restart.py | 32 - tests/features/test_watcher_routes.py | 64 ++ tests/regressions/test_scan_batching.py | 4 +- tox.ini | 10 + 79 files changed, 3500 insertions(+), 2417 deletions(-) create mode 100644 .github/workflows/ci-python.yml create mode 100644 .github/workflows/nightly.yml create mode 100644 .github/workflows/release-on-tag.yml create mode 100644 backend/features/index/watcher_scope.py delete mode 100644 backend/features/metadata/archived/native_extractors.py delete mode 100644 backend/routes/handlers/filesystem_new.py create mode 100644 backend/routes/handlers/releases.py delete mode 100644 js/components/ContextMenu.js create mode 100644 js/utils/dom.js create mode 100644 tests/features/test_open_in_folder.py create mode 100644 tests/features/test_upload_input.py delete mode 100644 tests/features/test_watcher_restart.py create mode 100644 tests/features/test_watcher_routes.py create mode 100644 tox.ini diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml new file mode 100644 index 00000000..2bbc3a51 --- /dev/null +++ b/.github/workflows/ci-python.yml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python: [3.10, 3.11] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest + - name: Run tests + run: | + python -m pytest -q diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..df3f2e57 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,34 @@ +name: Nightly build & prerelease +on: + schedule: + - cron: '0 2 * * *' # daily at 02:00 UTC + workflow_dispatch: {} + +permissions: + contents: write + +jobs: + nightly: + name: Nightly build and prerelease + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build nightly ZIP + run: | + TIMESTAMP=$(date -u +"%Y%m%d-%H%M") + ZIP=mjr-am-nightly-${TIMESTAMP}.zip + git archive --format=zip --output="$ZIP" HEAD || (zip -r "$ZIP" . -x .github/** tests/** __pycache__/**) + echo "ZIP=$ZIP" >> $GITHUB_ENV + + - name: Create nightly prerelease + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ env.ZIP }} + name: Nightly ${{ env.ZIP }} + body: Automated nightly build + prerelease: true + files: ${{ env.ZIP }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-on-tag.yml b/.github/workflows/release-on-tag.yml new file mode 100644 index 00000000..3f5051e7 --- /dev/null +++ b/.github/workflows/release-on-tag.yml @@ -0,0 +1,32 @@ +name: Release on tag +on: + push: + tags: + - 'v*.*.*' + +permissions: + contents: write + +jobs: + build-and-release: + name: Build ZIP and create Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build ZIP artifact + run: | + REF_NAME=${{ github.ref_name }} + ZIP_NAME=mjr-assets-manager-${REF_NAME}.zip + git archive --format=zip --output="$ZIP_NAME" ${{ github.ref }} || (zip -r "$ZIP_NAME" . -x .github/** tests/** __pycache__/**) + echo "ZIP_NAME=$ZIP_NAME" >> $GITHUB_ENV + + - name: Create GitHub Release and upload asset + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ github.ref_name }} + name: Release ${{ github.ref_name }} + files: ${{ env.ZIP_NAME }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index 2d370bf6..131e1559 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,7 +42,7 @@ ComfyUI-Majoor-AssetsManager/ adapters/ db/ # sqlite wrapper + schema + repos tools/ # exiftool/ffprobe wrappers - fs/ # watcher + scan helpers + fs/ # scan helpers features/ index/ # scan + search + incremental indexing metadata/ # extraction metadata + caching diff --git a/README.md b/README.md index cecb57fe..2fb55aa9 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,23 @@ Optional attribution request (non-binding): see `NOTICE`. ### ComfyUI Manager (recommended) Install via the ComfyUI Manager UI. +### Installing a specific version or nightly build +You can install a specific release tag or the nightly branch by using the GitHub ZIP archive URL. Example patterns: + +- Release tag (example): + + https://github.com/MajoorWaldi/ComfyUI-Majoor-AssetsManager/archive/refs/tags/v2.2.3.zip + +- Nightly / latest main branch (example): + + https://github.com/MajoorWaldi/ComfyUI-Majoor-AssetsManager/archive/refs/heads/main.zip + +If your ComfyUI registry or manager accepts a zip URL, paste one of the above URLs to install that specific ref. The extension manifest includes a `releases.zip_url_template` that tools can use to programmatically construct these URLs; the template is: + +`https://github.com/MajoorWaldi/ComfyUI-Majoor-AssetsManager/archive/refs/{ref}.zip` + +Use `{ref}` = `tags/vX.Y.Z` for a tag, or `{ref}` = `heads/` (for example `heads/main`) for branch installs (nightly). + ### Manual ```bash cd ComfyUI/custom_nodes @@ -181,6 +198,17 @@ Settings are stored under the `mjrSettings` key in `localStorage` (browser-side) Backend persistence is currently limited to `probeBackend.mode` (stored in the SQLite `metadata` table). It uses a version bump so changes propagate quickly even with multiple backend instances. Other settings are intentionally browser-local and won't sync across machines/browsers. - Page size (assets per request) + +## Compatibility + +This project officially supports Python 3.10 and 3.11. A CI matrix is provided in `.github/workflows/ci-python.yml` which runs the test-suite on both interpreters. + +Run tests across multiple Python versions locally with `tox` (requires those Python versions installed): + +```bash +pip install tox +tox +``` - Auto-scan (on open / on startup) - Status poll interval - Sidebar position (left/right) diff --git a/__init__.py b/__init__.py index edc1faa9..944e1e3b 100644 --- a/__init__.py +++ b/__init__.py @@ -186,7 +186,6 @@ try: "aiohttp", "PIL", # pillow "send2trash", - "watchdog", "aiosqlite", ] if os.name == "nt": diff --git a/backend/adapters/db/schema.py b/backend/adapters/db/schema.py index ee65be01..64aeb3a1 100644 --- a/backend/adapters/db/schema.py +++ b/backend/adapters/db/schema.py @@ -147,6 +147,8 @@ CREATE INDEX IF NOT EXISTS idx_assets_kind_mtime ON assets(kind, mtime); CREATE INDEX IF NOT EXISTS idx_assets_source ON assets(source); CREATE INDEX IF NOT EXISTS idx_assets_root_id ON assets(root_id); CREATE INDEX IF NOT EXISTS idx_assets_source_root_id ON assets(source, root_id); +-- Prevent duplicate entries for the same path/source/root (best-effort; filepath is already UNIQUE). +CREATE UNIQUE INDEX IF NOT EXISTS idx_assets_filepath_source_root ON assets(filepath, source, root_id); CREATE INDEX IF NOT EXISTS idx_metadata_rating ON asset_metadata(rating); CREATE INDEX IF NOT EXISTS idx_metadata_workflow_hash ON asset_metadata(workflow_hash); CREATE INDEX IF NOT EXISTS idx_metadata_quality_workflow ON asset_metadata(metadata_quality, has_workflow); @@ -319,7 +321,9 @@ async def _repair_asset_metadata_fts(db) -> Result[bool]: logger.warning("Repairing asset_metadata_fts (schema/triggers)") try: - async with db.atransaction(mode="immediate"): + async with db.atransaction(mode="immediate") as tx: + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") if needs_table_rebuild: await db.aexecutescript( """ @@ -374,6 +378,8 @@ async def _repair_asset_metadata_fts(db) -> Result[bool]: FROM asset_metadata; """ ) + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Commit failed") return Result.Ok(True) except Exception as exc: logger.warning("Failed to repair asset_metadata_fts: %s", exc) diff --git a/backend/adapters/db/sqlite.py b/backend/adapters/db/sqlite.py index eaf21452..9a810e06 100644 --- a/backend/adapters/db/sqlite.py +++ b/backend/adapters/db/sqlite.py @@ -25,6 +25,7 @@ import re import threading import time import uuid +import os from contextlib import contextmanager, asynccontextmanager from pathlib import Path from queue import Empty, Queue @@ -41,6 +42,8 @@ logger = get_logger(__name__) SQLITE_BUSY_TIMEOUT_MS = 5000 # Negative cache_size is in KiB. -64000 ~= 64 MiB cache. SQLITE_CACHE_SIZE_KIB = -64000 +ASSET_LOCKS_MAX = int(os.getenv("MAJOOR_ASSET_LOCKS_MAX", "10000") or 10000) +ASSET_LOCKS_TTL_S = float(os.getenv("MAJOOR_ASSET_LOCKS_TTL_SECONDS", "600") or 600.0) _TX_TOKEN: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("mjr_db_tx_token", default=None) _COLUMN_NAME_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)?$") @@ -311,7 +314,7 @@ class Sqlite: self._tx_state_lock = threading.Lock() # Thread-local storage for sync transactions (caller thread). self._tx_local = threading.local() - self._asset_locks: Dict[str, asyncio.Lock] = {} + self._asset_locks: Dict[str, Dict[str, Any]] = {} self._asset_locks_lock = threading.Lock() self.db_path.parent.mkdir(parents=True, exist_ok=True) @@ -407,14 +410,56 @@ class Sqlite: if self._write_lock is None: self._write_lock = asyncio.Lock() + def _prune_asset_locks_locked(self, now: float) -> None: + if not self._asset_locks: + return + cutoff = now - float(ASSET_LOCKS_TTL_S) + try: + for key, entry in list(self._asset_locks.items()): + try: + lock = entry.get("lock") + last = float(entry.get("last") or 0) + except Exception: + continue + try: + if lock is not None and getattr(lock, "locked", None) and lock.locked(): + continue + except Exception: + pass + if last and last < cutoff: + self._asset_locks.pop(key, None) + except Exception: + pass + + if ASSET_LOCKS_MAX > 0 and len(self._asset_locks) > ASSET_LOCKS_MAX: + try: + items = sorted(self._asset_locks.items(), key=lambda kv: float(kv[1].get("last") or 0)) + to_drop = max(0, len(items) - ASSET_LOCKS_MAX) + for key, entry in items[:to_drop]: + try: + lock = entry.get("lock") + if lock is not None and getattr(lock, "locked", None) and lock.locked(): + continue + except Exception: + pass + self._asset_locks.pop(key, None) + except Exception: + pass + def _get_or_create_asset_lock(self, asset_id: Any) -> asyncio.Lock: key = _asset_lock_key(asset_id) + now = time.time() with self._asset_locks_lock: - lock = self._asset_locks.get(key) - if lock: - return lock + entry = self._asset_locks.get(key) + if entry: + try: + entry["last"] = now + return entry["lock"] + except Exception: + pass lock = asyncio.Lock() - self._asset_locks[key] = lock + self._asset_locks[key] = {"lock": lock, "last": now} + self._prune_asset_locks_locked(now) return lock @asynccontextmanager @@ -452,6 +497,18 @@ class Sqlite: pass return out + async def _with_query_timeout(self, coro): + try: + timeout = float(self._query_timeout or 0) + except Exception: + timeout = 0.0 + if timeout > 0: + try: + return await asyncio.wait_for(coro, timeout=timeout) + except asyncio.TimeoutError: + return Result.Err(ErrorCode.TIMEOUT, "Database operation timed out") + return await coro + async def _execute_async( self, query: str, @@ -497,36 +554,38 @@ class Sqlite: commit: bool, tx_token: Optional[str] = None, ) -> Result[Any]: - try: - lock = self._write_lock - is_write = self._is_write_sql(query) - if is_write and lock is not None: - # BEGIN acquires the write lock already; avoid deadlocking by reacquiring it - # for statements executed within the same transaction. - if tx_token: - try: - with self._tx_state_lock: - in_tx = tx_token in self._tx_write_lock_tokens - except Exception: + async def _execute_inner() -> Result[Any]: + try: + lock = self._write_lock + is_write = self._is_write_sql(query) + if is_write and lock is not None: + # BEGIN acquires the write lock already; avoid deadlocking by reacquiring it + # for statements executed within the same transaction. + if tx_token: + try: + with self._tx_state_lock: + in_tx = tx_token in self._tx_write_lock_tokens + except Exception: + in_tx = False + else: in_tx = False - else: - in_tx = False - if in_tx: - return await self._execute_on_conn_locked_async(conn, query, params, fetch, commit=commit) - async with lock: - return await self._execute_on_conn_locked_async(conn, query, params, fetch, commit=commit) - return await self._execute_on_conn_locked_async(conn, query, params, fetch, commit=commit) - except sqlite3.IntegrityError as exc: - logger.warning("Integrity error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, f"Integrity error: {exc}") - except sqlite3.OperationalError as exc: - if "interrupted" in str(exc).lower(): - return Result.Err(ErrorCode.TIMEOUT, "Database operation interrupted (query timeout)") - logger.error("Operational error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, f"Operational error: {exc}") - except Exception as exc: - logger.error("Unexpected database error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, str(exc)) + if in_tx: + return await self._execute_on_conn_locked_async(conn, query, params, fetch, commit=commit) + async with lock: + return await self._execute_on_conn_locked_async(conn, query, params, fetch, commit=commit) + return await self._execute_on_conn_locked_async(conn, query, params, fetch, commit=commit) + except sqlite3.IntegrityError as exc: + logger.warning("Integrity error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, f"Integrity error: {exc}") + except sqlite3.OperationalError as exc: + if "interrupted" in str(exc).lower(): + return Result.Err(ErrorCode.TIMEOUT, "Database operation interrupted (query timeout)") + logger.error("Operational error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, f"Operational error: {exc}") + except Exception as exc: + logger.error("Unexpected database error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, str(exc)) + return await self._with_query_timeout(_execute_inner()) async def _execute_on_conn_locked_async( self, @@ -687,33 +746,35 @@ class Sqlite: commit: bool, tx_token: Optional[str] = None, ) -> Result[int]: - try: - lock = self._write_lock - is_write = self._is_write_sql(query) - if is_write and lock is not None: - # Check if we're inside an existing transaction that holds the write lock. - # Use _tx_state_lock to avoid race conditions with concurrent rollback/commit. - if tx_token: - try: - with self._tx_state_lock: - in_tx = tx_token in self._tx_write_lock_tokens - except Exception: + async def _execute_inner() -> Result[int]: + try: + lock = self._write_lock + is_write = self._is_write_sql(query) + if is_write and lock is not None: + # Check if we're inside an existing transaction that holds the write lock. + # Use _tx_state_lock to avoid race conditions with concurrent rollback/commit. + if tx_token: + try: + with self._tx_state_lock: + in_tx = tx_token in self._tx_write_lock_tokens + except Exception: + in_tx = False + else: in_tx = False - else: - in_tx = False - if in_tx: - return await self._executemany_on_conn_locked_async(conn, query, params_list, commit=commit) - async with lock: - return await self._executemany_on_conn_locked_async(conn, query, params_list, commit=commit) - return await self._executemany_on_conn_locked_async(conn, query, params_list, commit=commit) - except sqlite3.OperationalError as exc: - if "interrupted" in str(exc).lower(): - return Result.Err(ErrorCode.TIMEOUT, "Database operation interrupted (query timeout)") - logger.error("Batch execute error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, str(exc)) - except Exception as exc: - logger.error("Batch execute error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, str(exc)) + if in_tx: + return await self._executemany_on_conn_locked_async(conn, query, params_list, commit=commit) + async with lock: + return await self._executemany_on_conn_locked_async(conn, query, params_list, commit=commit) + return await self._executemany_on_conn_locked_async(conn, query, params_list, commit=commit) + except sqlite3.OperationalError as exc: + if "interrupted" in str(exc).lower(): + return Result.Err(ErrorCode.TIMEOUT, "Database operation interrupted (query timeout)") + logger.error("Batch execute error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, str(exc)) + except Exception as exc: + logger.error("Batch execute error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, str(exc)) + return await self._with_query_timeout(_execute_inner()) async def _executemany_on_conn_locked_async( self, conn: aiosqlite.Connection, query: str, params_list: List[Tuple], *, commit: bool @@ -774,26 +835,28 @@ class Sqlite: await self._release_connection_async(conn) async def _executescript_on_conn_async(self, conn: aiosqlite.Connection, script: str, *, commit: bool) -> Result[bool]: - try: - for attempt in range(self._lock_retry_attempts + 1): - try: - await conn.executescript(script) - if commit: - await conn.commit() - return Result.Ok(True) - except sqlite3.OperationalError as exc: - if self._is_locked_error(exc) and attempt < self._lock_retry_attempts: - await self._sleep_backoff(attempt) - continue - raise - except sqlite3.OperationalError as exc: - if "interrupted" in str(exc).lower(): - return Result.Err(ErrorCode.TIMEOUT, "Database operation interrupted (query timeout)") - logger.error("Script execution error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, str(exc)) - except Exception as exc: - logger.error("Script execution error: %s", exc) - return Result.Err(ErrorCode.DB_ERROR, str(exc)) + async def _execute_inner() -> Result[bool]: + try: + for attempt in range(self._lock_retry_attempts + 1): + try: + await conn.executescript(script) + if commit: + await conn.commit() + return Result.Ok(True) + except sqlite3.OperationalError as exc: + if self._is_locked_error(exc) and attempt < self._lock_retry_attempts: + await self._sleep_backoff(attempt) + continue + raise + except sqlite3.OperationalError as exc: + if "interrupted" in str(exc).lower(): + return Result.Err(ErrorCode.TIMEOUT, "Database operation interrupted (query timeout)") + logger.error("Script execution error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, str(exc)) + except Exception as exc: + logger.error("Script execution error: %s", exc) + return Result.Err(ErrorCode.DB_ERROR, str(exc)) + return await self._with_query_timeout(_execute_inner()) def executescript(self, script: str) -> Result[bool]: """Execute a multi-statement SQL script (sync).""" @@ -1190,17 +1253,22 @@ class Sqlite: def transaction(self, mode: str = "immediate"): """Context manager for a DB transaction (sync).""" # Back-compat sync transaction: routes token via thread-local. + tx_state = Result.Ok(True) begin_res = self._loop_thread.run(self._begin_tx_async(mode)) if not begin_res.ok or not begin_res.data: - raise RuntimeError(str(begin_res.error or "Failed to begin transaction")) + tx_state = Result.Err(ErrorCode.DB_ERROR, str(begin_res.error or "Failed to begin transaction")) + yield tx_state + return token = str(begin_res.data) self._tx_local.token = token try: - yield + yield tx_state commit_res = self._loop_thread.run(self._commit_tx_async(token)) if not commit_res.ok: - raise RuntimeError(str(commit_res.error or "Commit failed")) + tx_state.ok = False + tx_state.code = str(getattr(commit_res, "code", None) or ErrorCode.DB_ERROR) + tx_state.error = str(commit_res.error or "Commit failed") except Exception: try: self._loop_thread.run(self._rollback_tx_async(token)) @@ -1217,17 +1285,22 @@ class Sqlite: @asynccontextmanager async def atransaction(self, mode: str = "immediate"): """Async context manager for a DB transaction.""" + tx_state = Result.Ok(True) begin_res = await self._begin_tx_for_async(mode) if not begin_res.ok or not begin_res.data: - raise RuntimeError(str(begin_res.error or "Failed to begin transaction")) + tx_state = Result.Err(ErrorCode.DB_ERROR, str(begin_res.error or "Failed to begin transaction")) + yield tx_state + return token = str(begin_res.data) token_handle = _TX_TOKEN.set(token) try: - yield + yield tx_state commit_res = await self._commit_tx_for_async(token) if not commit_res.ok: - raise RuntimeError(str(commit_res.error or "Commit failed")) + tx_state.ok = False + tx_state.code = str(getattr(commit_res, "code", None) or ErrorCode.DB_ERROR) + tx_state.error = str(commit_res.error or "Commit failed") except Exception: try: await self._rollback_tx_for_async(token) diff --git a/backend/adapters/fs/__init__.py b/backend/adapters/fs/__init__.py index 63f85f0e..d2e8bc80 100644 --- a/backend/adapters/fs/__init__.py +++ b/backend/adapters/fs/__init__.py @@ -1,4 +1,4 @@ """ -Filesystem adapters (watchers, scan helpers, etc.). +Filesystem adapters (scan helpers, etc.). """ diff --git a/backend/config.py b/backend/config.py index 6a687627..11cb65d2 100644 --- a/backend/config.py +++ b/backend/config.py @@ -6,7 +6,7 @@ import sys import logging from pathlib import Path -from .utils import env_bool, env_float +from .utils import env_float logger = logging.getLogger(__name__) @@ -67,24 +67,6 @@ def _env_int(name: str, default: int) -> int: except (TypeError, ValueError): return default -# [CHANGED] Default to True to enable real-time updates out-of-the-box -ENABLE_FILE_WATCHER = env_bool("MAJOOR_ENABLE_FILE_WATCHER", True) -WATCHER_INTERVAL_SECONDS = env_float("MAJOOR_WATCHER_INTERVAL", 15.0) -WATCHER_JOIN_TIMEOUT = env_float("MAJOOR_WATCHER_JOIN_TIMEOUT", 5.0) -WATCHER_PATHS = [] -for raw_path in os.getenv("MAJOOR_WATCHER_PATHS", OUTPUT_ROOT).split(os.pathsep): - cleaned = raw_path.strip() - if not cleaned: - continue - try: - WATCHER_PATHS.append(Path(cleaned).resolve()) - except (OSError, RuntimeError, ValueError): - logger.warning("Invalid path in MAJOOR_WATCHER_PATHS: %r", raw_path) - continue - -if not WATCHER_PATHS: - WATCHER_PATHS = [OUTPUT_ROOT_PATH] - # Tool timeouts EXIFTOOL_TIMEOUT = _env_int("MAJOOR_EXIFTOOL_TIMEOUT", 15) FFPROBE_TIMEOUT = _env_int("MAJOOR_FFPROBE_TIMEOUT", 10) @@ -95,19 +77,32 @@ DB_MAX_CONNECTIONS = _env_int("MAJOOR_DB_MAX_CONNECTIONS", 8) DB_QUERY_TIMEOUT = env_float("MAJOOR_DB_QUERY_TIMEOUT", 30.0) TO_THREAD_TIMEOUT_S = env_float("MAJOOR_TO_THREAD_TIMEOUT", 30.0) MAX_METADATA_JSON_BYTES = _env_int("MAJOOR_MAX_METADATA_JSON_BYTES", 2 * 1024 * 1024) # 2MB +METADATA_CACHE_MAX = _env_int("MAJOOR_METADATA_CACHE_MAX", 100_000) +METADATA_CACHE_TTL_SECONDS = env_float("MAJOOR_METADATA_CACHE_TTL_SECONDS", 90.0 * 24.0 * 3600.0) +METADATA_CACHE_CLEANUP_INTERVAL_SECONDS = env_float("MAJOOR_METADATA_CACHE_CLEANUP_INTERVAL_SECONDS", 300.0) +METADATA_EXTRACT_CONCURRENCY = _env_int("MAJOOR_METADATA_EXTRACT_CONCURRENCY", 1) + +# Index dedupe (avoid double-indexing bursts from multiple event sources) +INDEX_DEDUPE_TTL_SECONDS = env_float("MAJOOR_INDEX_DEDUPE_TTL_SECONDS", 2.0) +INDEX_DEDUPE_MAX = _env_int("MAJOOR_INDEX_DEDUPE_MAX", 5000) + +# File watcher (watches for manual file additions in output/custom directories) +# Disable with MJR_ENABLE_WATCHER=0 +WATCHER_ENABLED = os.getenv("MJR_ENABLE_WATCHER", "1").strip().lower() not in ("0", "false", "no", "off") +WATCHER_DEBOUNCE_MS = _env_int("MJR_WATCHER_DEBOUNCE_MS", 500) +WATCHER_DEDUPE_TTL_MS = _env_int("MJR_WATCHER_DEDUPE_TTL_MS", 3000) # Background scan / filesystem listing tuning BG_SCAN_FAILURE_HISTORY_MAX = _env_int("MAJOOR_BG_SCAN_FAILURE_HISTORY_MAX", 50) SCAN_PENDING_MAX = _env_int("MAJOOR_SCAN_PENDING_MAX", 64) MANUAL_BG_SCAN_GRACE_SECONDS = env_float("MAJOOR_MANUAL_BG_SCAN_GRACE_SECONDS", 30.0) +BG_SCAN_ON_LIST = os.getenv("MAJOOR_BG_SCAN_ON_LIST", "0").strip().lower() in ("1", "true", "yes", "on") SEARCH_MAX_LIMIT = _env_int("MAJOOR_SEARCH_MAX_LIMIT", 500) SEARCH_MAX_OFFSET = _env_int("MAJOOR_SEARCH_MAX_OFFSET", 10_000) # Filesystem listing cache (used by filesystem fallback search/list) FS_LIST_CACHE_MAX = _env_int("MAJOOR_FS_LIST_CACHE_MAX", 32) FS_LIST_CACHE_TTL_SECONDS = env_float("MAJOOR_FS_LIST_CACHE_TTL_SECONDS", 1.5) -FS_LIST_CACHE_WATCHDOG = env_bool("MAJOOR_FS_LIST_CACHE_WATCHDOG", True) - # Scanner batching (bounded transactions) # Tweak only if you know your workload; larger batches reduce transaction overhead but increase lock time. SCAN_BATCH_SMALL_THRESHOLD = _env_int("MAJOOR_SCAN_BATCH_SMALL_THRESHOLD", 100) diff --git a/backend/custom_roots.py b/backend/custom_roots.py index e489cd07..71db4909 100644 --- a/backend/custom_roots.py +++ b/backend/custom_roots.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional from uuid import uuid4 from .shared import Result, get_logger -from .config import INDEX_DIR +from .config import INDEX_DIR, OUTPUT_ROOT logger = get_logger(__name__) @@ -50,6 +50,16 @@ def _is_symlink_like(path: Path) -> bool: return False +def _path_is_relative_to(path: Path, root: Path) -> bool: + try: + if hasattr(path, "is_relative_to"): + return path.is_relative_to(root) # type: ignore[attr-defined] + path.relative_to(root) + return True + except Exception: + return False + + def _normalize_dir_path(path: str) -> Optional[Path]: if not path or "\x00" in path: return None @@ -114,11 +124,14 @@ def list_custom_roots() -> Result[List[Dict[str, Any]]]: path = str(r.get("path") or "").strip() if not rid or not path: continue + normalized = _normalize_dir_path(path) + if not normalized: + continue cleaned.append( { "id": rid, - "path": path, - "label": str(r.get("label") or "").strip() or Path(path).name or path, + "path": str(normalized), + "label": str(r.get("label") or "").strip() or normalized.name or str(normalized), "created_at": r.get("created_at"), } ) @@ -140,6 +153,26 @@ def add_custom_root(path: str, label: Optional[str] = None) -> Result[Dict[str, created_at = _utc_now_iso() safe_label = (label or "").strip() or normalized.name or resolved + output_root = None + input_root = None + try: + if OUTPUT_ROOT: + output_root = Path(OUTPUT_ROOT).resolve() + except Exception: + output_root = None + try: + import folder_paths + input_root = Path(folder_paths.get_input_directory()).resolve() + except Exception: + input_root = None + + if output_root: + if _path_is_relative_to(normalized, output_root) or _path_is_relative_to(output_root, normalized): + return Result.Err("OVERLAP", f"Root overlaps with output: {output_root}") + if input_root: + if _path_is_relative_to(normalized, input_root) or _path_is_relative_to(input_root, normalized): + return Result.Err("OVERLAP", f"Root overlaps with input: {input_root}") + with _LOCK: store = _read_store() roots = store.get("roots") or [] @@ -158,6 +191,13 @@ def add_custom_root(path: str, label: Optional[str] = None) -> Result[Dict[str, "already_exists": True, } ) + if existing_path: + try: + existing_resolved = Path(str(existing_path)).expanduser().resolve(strict=False) + if _path_is_relative_to(normalized, existing_resolved) or _path_is_relative_to(existing_resolved, normalized): + return Result.Err("OVERLAP", f"Root overlaps with existing: {existing_resolved}") + except Exception: + continue roots.append({"id": root_id, "path": resolved, "label": safe_label, "created_at": created_at}) store["roots"] = roots diff --git a/backend/deps.py b/backend/deps.py index d472e781..2737637a 100644 --- a/backend/deps.py +++ b/backend/deps.py @@ -3,16 +3,19 @@ Dependency injection - builds services. Simple, debug-friendly DI without framework magic. """ +import asyncio import os +from pathlib import Path -from .shared import get_logger, log_success +from .shared import Result, get_logger, log_success from .adapters.db.sqlite import Sqlite from .adapters.db.schema import init_schema, migrate_schema, table_has_column from .adapters.tools import ExifTool, FFProbe from .features.metadata import MetadataService from .features.health import HealthService from .features.index import IndexService -from .features.index.watcher import DirectoryWatcher +from .features.index.watcher import OutputWatcher +from .features.index.watcher_scope import load_watcher_scope, build_watch_paths from .features.tags import RatingTagsSyncWorker from .settings import AppSettings from .config import ( @@ -21,17 +24,14 @@ from .config import ( EXIFTOOL_TIMEOUT, FFPROBE_BIN, FFPROBE_TIMEOUT, - ENABLE_FILE_WATCHER, - WATCHER_PATHS, - WATCHER_INTERVAL_SECONDS, - WATCHER_JOIN_TIMEOUT, DB_TIMEOUT, DB_MAX_CONNECTIONS, + WATCHER_ENABLED, ) logger = get_logger(__name__) -async def build_services(db_path: str = None) -> dict: +async def build_services(db_path: str = None) -> Result[dict]: """ Build all services (DI container). @@ -39,7 +39,7 @@ async def build_services(db_path: str = None) -> dict: db_path: Path to SQLite database (default: from config.INDEX_DB) Returns: - Dict of service instances + Result[dict] of service instances """ logger.info("Building services...") @@ -49,13 +49,17 @@ async def build_services(db_path: str = None) -> dict: # Initialize database logger.info(f"Initializing database: {db_path}") - db = Sqlite(db_path, max_connections=DB_MAX_CONNECTIONS, timeout=DB_TIMEOUT) + try: + db = Sqlite(db_path, max_connections=DB_MAX_CONNECTIONS, timeout=DB_TIMEOUT) + except Exception as exc: + logger.error("Failed to initialize database: %s", exc) + return Result.Err("DB_ERROR", f"Failed to initialize database: {exc}") # Run migrations migrate_result = await migrate_schema(db) if not migrate_result.ok: logger.error(f"Schema migration failed: {migrate_result.error}") - raise RuntimeError(f"Failed to initialize database: {migrate_result.error}") + return Result.Err(migrate_result.code or "DB_ERROR", f"Failed to initialize database: {migrate_result.error}") # Initialize adapters exiftool = ExifTool(bin_name=EXIFTOOL_BIN, timeout=EXIFTOOL_TIMEOUT) @@ -104,6 +108,10 @@ async def build_services(db_path: str = None) -> dict: "index": index_service, "settings": settings_service, } + try: + services["watcher_scope"] = await load_watcher_scope(db) + except Exception: + services["watcher_scope"] = {"scope": "output", "custom_root_id": ""} # Best-effort filesystem sync for rating/tags (sidecar / ExifTool). try: @@ -111,15 +119,48 @@ async def build_services(db_path: str = None) -> dict: except Exception as exc: logger.debug("RatingTagsSyncWorker disabled: %s", exc) - if ENABLE_FILE_WATCHER and WATCHER_PATHS: - watcher = DirectoryWatcher( - index_service, - WATCHER_PATHS, - WATCHER_INTERVAL_SECONDS, - WATCHER_JOIN_TIMEOUT - ) - watcher.start() - services["watcher"] = watcher + # File watcher for manual additions (disabled by default, enable with MJR_ENABLE_WATCHER=1) + if WATCHER_ENABLED: + try: + watcher = await _create_watcher(index_service) + services["watcher"] = watcher + log_success(logger, "File watcher enabled") + except Exception as exc: + logger.warning("File watcher disabled: %s", exc) log_success(logger, "All services initialized") - return services + return Result.Ok(services) + + +async def _create_watcher(index_service: IndexService) -> OutputWatcher: + """Create and start the file watcher for output directories.""" + + async def index_callback(filepaths: list, base_dir: str, source: str | None = None, root_id: str | None = None): + """Called by watcher when files are ready to index.""" + if not filepaths: + return + paths = [Path(f) for f in filepaths if f] + if paths: + await index_service.index_paths( + paths=paths, + base_dir=base_dir, + incremental=True, + source=source or "watcher", + root_id=root_id, + ) + + watcher = OutputWatcher(index_callback) + + # Collect directories to watch + try: + scope_cfg = await load_watcher_scope(index_service.db) + except Exception: + scope_cfg = {"scope": "output", "custom_root_id": ""} + + watch_paths = build_watch_paths(scope_cfg.get("scope"), scope_cfg.get("custom_root_id")) + + if watch_paths: + loop = asyncio.get_event_loop() + await watcher.start(watch_paths, loop) + + return watcher diff --git a/backend/features/health/service.py b/backend/features/health/service.py index 80d6a151..f7daf238 100644 --- a/backend/features/health/service.py +++ b/backend/features/health/service.py @@ -215,6 +215,12 @@ class HealthService: ) last_scan_end = last_scan_result.data[0]["value"] if last_scan_result.ok and last_scan_result.data else None + last_index_result = await self.db.aexecute( + "SELECT value FROM metadata WHERE key = 'last_index_end'", + fetch=True + ) + last_index_end = last_index_result.data[0]["value"] if last_index_result.ok and last_index_result.data else None + # Get counts by kind by_kind = {} if kind_result.ok and kind_result.data and len(kind_result.data) > 0: @@ -234,6 +240,7 @@ class HealthService: "with_workflows": workflow_count, "with_generation_data": generation_count, "last_scan_end": last_scan_end, + "last_index_end": last_index_end, "tool_availability": self._get_tool_capabilities(), "tool_paths": get_tool_paths(), } diff --git a/backend/features/index/enricher.py b/backend/features/index/enricher.py index 6264d21c..756f001c 100644 --- a/backend/features/index/enricher.py +++ b/backend/features/index/enricher.py @@ -5,9 +5,9 @@ from __future__ import annotations import os import asyncio -from typing import List, Dict, Any, Optional, Tuple +from typing import List, Dict, Any, Optional -from ...shared import get_logger +from ...shared import get_logger, Result from ...adapters.db.sqlite import Sqlite from ..metadata import MetadataService @@ -28,7 +28,6 @@ class MetadataEnricher: self, db: Sqlite, metadata_service: MetadataService, - scan_lock: asyncio.Lock, compute_state_hash_fn, prepare_metadata_fields_fn, metadata_error_payload_fn, @@ -39,14 +38,12 @@ class MetadataEnricher: Args: db: Database adapter instance metadata_service: Metadata service for extraction - scan_lock: Shared lock for database writes compute_state_hash_fn: Function to compute state hash prepare_metadata_fields_fn: Function to prepare metadata fields metadata_error_payload_fn: Function to create error payload """ self.db = db self.metadata = metadata_service - self._scan_lock = scan_lock self._compute_state_hash = compute_state_hash_fn self._prepare_metadata_fields = prepare_metadata_fields_fn self._metadata_error_payload = metadata_error_payload_fn @@ -124,8 +121,7 @@ class MetadataEnricher: if fp and aid: id_by_fp[str(fp)] = aid - asset_updates: List[Tuple[Any, Any, Any, int]] = [] - meta_updates: List[Tuple[int, Any]] = [] + updates: List[Dict[str, Any]] = [] # Import helper locally to use shared logic (including FTS population) from .metadata_helpers import MetadataHelpers @@ -159,28 +155,76 @@ class MetadataEnricher: height = meta.get("height") duration = meta.get("duration") - asset_updates.append((width, height, duration, asset_id)) - meta_updates.append((asset_id, metadata_result)) + updates.append( + { + "asset_id": asset_id, + "width": width, + "height": height, + "duration": duration, + "metadata_result": metadata_result, + "filepath": fp, + } + ) - if not asset_updates and not meta_updates: + if not updates: return - # Keep DB writes serialized with scans/index_files to avoid lock contention. - async with self._scan_lock: - async with self.db.atransaction(mode="immediate"): - if asset_updates: - await self.db.aexecutemany( - """ - UPDATE assets - SET width = COALESCE(?, width), - height = COALESCE(?, height), - duration = COALESCE(?, duration), - updated_at = CURRENT_TIMESTAMP - WHERE id = ? - """, - asset_updates, - ) - if meta_updates: - # Use shared helper to write metadata (ensures tags_text/FTS is populated with prompt/geninfo) - for asset_id, meta_res in meta_updates: - await MetadataHelpers.write_asset_metadata_row(self.db, asset_id, meta_res) + for item in updates: + asset_id = item.get("asset_id") + if not asset_id: + continue + meta_res = item.get("metadata_result") + if not isinstance(meta_res, Result): + continue + try: + async with self.db.lock_for_asset(asset_id): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + logger.warning( + "Metadata enrichment skipped (transaction begin failed) for asset_id=%s: %s", + asset_id, + tx.error, + ) + continue + await self.db.aexecute( + """ + UPDATE assets + SET width = COALESCE(?, width), + height = COALESCE(?, height), + duration = COALESCE(?, duration), + updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (item.get("width"), item.get("height"), item.get("duration"), asset_id), + ) + # Use shared helper to write metadata (ensures tags_text/FTS is populated with prompt/geninfo) + await MetadataHelpers.write_asset_metadata_row( + self.db, + asset_id, + meta_res, + filepath=item.get("filepath"), + ) + if not tx.ok: + logger.warning("Metadata enrichment commit failed for asset_id=%s: %s", asset_id, tx.error) + else: + # Notify frontend so workflow dot updates promptly. + try: + res = await self.db.aquery( + """ + SELECT a.id, m.has_workflow AS has_workflow, + m.has_generation_data AS has_generation_data + FROM assets a + LEFT JOIN asset_metadata m ON a.id = m.asset_id + WHERE a.id = ? + LIMIT 1 + """, + (int(asset_id),), + ) + if res.ok and res.data: + payload = dict(res.data[0]) + from ...routes.registry import PromptServer + PromptServer.instance.send_sync("mjr-asset-updated", payload) + except Exception: + pass + except Exception as exc: + logger.warning("Metadata enrichment update failed for asset_id=%s: %s", asset_id, exc) diff --git a/backend/features/index/metadata_helpers.py b/backend/features/index/metadata_helpers.py index 5ce86ea2..98bee41d 100644 --- a/backend/features/index/metadata_helpers.py +++ b/backend/features/index/metadata_helpers.py @@ -4,10 +4,17 @@ Metadata helpers - shared utilities for metadata operations. import hashlib import asyncio import json +import time +from datetime import datetime, timedelta, UTC from typing import Dict, Any, Tuple, Optional -from ...config import MAX_METADATA_JSON_BYTES -from ...shared import Result +from ...config import ( + MAX_METADATA_JSON_BYTES, + METADATA_CACHE_MAX, + METADATA_CACHE_TTL_SECONDS, + METADATA_CACHE_CLEANUP_INTERVAL_SECONDS, +) +from ...shared import Result, get_logger from ...adapters.db.sqlite import Sqlite from ..metadata.parsing_utils import ( looks_like_comfyui_workflow, @@ -16,6 +23,9 @@ from ..metadata.parsing_utils import ( ) MAX_TAG_LENGTH = 100 +logger = get_logger(__name__) +_METADATA_CACHE_CLEANUP_LOCK = asyncio.Lock() +_METADATA_CACHE_LAST_CLEANUP = 0.0 class MetadataHelpers: @@ -27,7 +37,7 @@ class MetadataHelpers: """ @staticmethod - def prepare_metadata_fields(metadata_result: Result[Dict[str, Any]]) -> Tuple[bool, bool, str, str]: + def prepare_metadata_fields(metadata_result: Result[Dict[str, Any]]) -> Tuple[Optional[bool], Optional[bool], str, str]: """ Extract the workflow/generation metadata attributes and JSON payload. @@ -37,8 +47,8 @@ class MetadataHelpers: Returns: Tuple of (has_workflow, has_generation_data, metadata_quality, metadata_raw_json) """ - has_workflow = False - has_generation_data = False + has_workflow: Optional[bool] = None + has_generation_data: Optional[bool] = None metadata_quality = "none" metadata_raw_json = "{}" @@ -90,7 +100,7 @@ class MetadataHelpers: return parsed if isinstance(parsed, dict) else None return None - if metadata_result and metadata_result.ok and metadata_result.data: + if metadata_result and metadata_result.ok and isinstance(metadata_result.data, dict) and metadata_result.data: meta = metadata_result.data raw_workflow = meta.get("workflow") @@ -129,6 +139,14 @@ class MetadataHelpers: return has_workflow, has_generation_data, metadata_quality, metadata_raw_json + @staticmethod + def _bool_to_db(value: Optional[bool]) -> Optional[int]: + if value is True: + return 1 + if value is False: + return 0 + return None + @staticmethod def metadata_error_payload(metadata_result: Result[Dict[str, Any]], filepath: str) -> Result[Dict[str, Any]]: """ @@ -159,7 +177,12 @@ class MetadataHelpers: return Result.Ok(payload, quality=quality) @staticmethod - async def write_asset_metadata_row(db: Sqlite, asset_id: int, metadata_result: Result[Dict[str, Any]]) -> Result[Any]: + async def write_asset_metadata_row( + db: Sqlite, + asset_id: int, + metadata_result: Result[Dict[str, Any]], + filepath: Optional[str] = None, + ) -> Result[Any]: """ Insert or update the asset_metadata row with the latest metadata flags. @@ -172,18 +195,24 @@ class MetadataHelpers: Result from database operation """ has_workflow, has_generation_data, metadata_quality, metadata_raw_json = MetadataHelpers.prepare_metadata_fields(metadata_result) + db_has_workflow = MetadataHelpers._bool_to_db(has_workflow) + db_has_generation = MetadataHelpers._bool_to_db(has_generation_data) # Guard against pathological/huge JSON payloads; keep DB stable. try: max_bytes = int(MAX_METADATA_JSON_BYTES or 0) except Exception: max_bytes = 0 + truncated = False + original_bytes = 0 if max_bytes and isinstance(metadata_raw_json, str): try: nbytes = len(metadata_raw_json.encode("utf-8", errors="ignore")) except Exception: nbytes = 0 if nbytes > max_bytes: + truncated = True + original_bytes = int(nbytes) try: metadata_raw_json = json.dumps( {"_truncated": True, "original_bytes": int(nbytes)}, @@ -192,6 +221,27 @@ class MetadataHelpers: ) except Exception: metadata_raw_json = "{}" + if truncated: + try: + metadata_result.meta["truncated"] = True + metadata_result.meta["original_bytes"] = int(original_bytes) + metadata_result.meta["max_bytes"] = int(max_bytes) + except Exception: + pass + try: + fp = filepath + if not fp and metadata_result and isinstance(metadata_result.data, dict): + fp = metadata_result.data.get("filepath") + suffix = f" filepath={fp}" if isinstance(fp, str) and fp else "" + logger.warning( + "Metadata JSON truncated for asset_id=%s%s (bytes=%s > max=%s)", + asset_id, + suffix, + original_bytes, + max_bytes, + ) + except Exception: + pass extracted_rating = 0 extracted_tags_json = "[]" @@ -333,8 +383,8 @@ class MetadataHelpers: extracted_rating, extracted_tags_json, extracted_tags_text, - 1 if has_workflow else 0, - 1 if has_generation_data else 0, + db_has_workflow, + db_has_generation, metadata_quality, metadata_raw_json, ), @@ -379,17 +429,19 @@ class MetadataHelpers: current = result.data[0] if result.data else None new_has_workflow, new_has_generation_data, _, new_metadata_raw = MetadataHelpers.prepare_metadata_fields(metadata_result) + new_has_workflow_db = MetadataHelpers._bool_to_db(new_has_workflow) + new_has_generation_db = MetadataHelpers._bool_to_db(new_has_generation_data) - current_has_workflow = current["has_workflow"] if current else 0 - current_has_generation = current["has_generation_data"] if current else 0 + current_has_workflow = current.get("has_workflow") if current else None + current_has_generation = current.get("has_generation_data") if current else None current_raw = current["metadata_raw"] if current else "" - if (current_has_workflow == (1 if new_has_workflow else 0) and - current_has_generation == (1 if new_has_generation_data else 0) and + if (current_has_workflow == new_has_workflow_db and + current_has_generation == new_has_generation_db and current_raw == new_metadata_raw): return False - write_result = await MetadataHelpers.write_asset_metadata_row(db, asset_id, metadata_result) + write_result = await MetadataHelpers.write_asset_metadata_row(db, asset_id, metadata_result, filepath=filepath) if write_result.ok: # We assume write_journal_fn is async now await write_journal_fn(filepath, base_dir, state_hash, mtime, size) @@ -429,6 +481,71 @@ class MetadataHelpers: except json.JSONDecodeError: return None + @staticmethod + async def _maybe_cleanup_metadata_cache(db: Sqlite) -> None: + global _METADATA_CACHE_LAST_CLEANUP + try: + max_entries = int(METADATA_CACHE_MAX or 0) + except Exception: + max_entries = 0 + try: + ttl_seconds = float(METADATA_CACHE_TTL_SECONDS or 0) + except Exception: + ttl_seconds = 0.0 + try: + interval = float(METADATA_CACHE_CLEANUP_INTERVAL_SECONDS or 0) + except Exception: + interval = 0.0 + + if max_entries <= 0 and ttl_seconds <= 0: + return + + now = time.time() + if interval > 0 and (now - _METADATA_CACHE_LAST_CLEANUP) < interval: + return + + async with _METADATA_CACHE_CLEANUP_LOCK: + now = time.time() + if interval > 0 and (_METADATA_CACHE_LAST_CLEANUP and (now - _METADATA_CACHE_LAST_CLEANUP) < interval): + return + _METADATA_CACHE_LAST_CLEANUP = now + + if ttl_seconds > 0: + try: + cutoff = datetime.now(UTC) - timedelta(seconds=ttl_seconds) + cutoff_str = cutoff.strftime("%Y-%m-%d %H:%M:%S") + await db.aexecute( + "DELETE FROM metadata_cache WHERE last_updated < ?", + (cutoff_str,), + ) + except Exception: + pass + + if max_entries > 0: + try: + count_res = await db.aquery("SELECT COUNT(1) AS count FROM metadata_cache") + if count_res.ok and count_res.data: + total = int(count_res.data[0].get("count") or 0) + else: + total = 0 + if total > max_entries: + to_remove = max(0, total - max_entries) + if to_remove: + await db.aexecute( + """ + DELETE FROM metadata_cache + WHERE filepath IN ( + SELECT filepath + FROM metadata_cache + ORDER BY last_updated ASC + LIMIT ? + ) + """, + (to_remove,), + ) + except Exception: + pass + @staticmethod async def store_metadata_cache(db: Sqlite, filepath: str, state_hash: str, metadata_result: Result[Dict[str, Any]]) -> Result[Any]: """ @@ -462,10 +579,24 @@ class MetadataHelpers: except Exception: nbytes = 0 if nbytes > max_bytes: - return Result.Err("CACHE_SKIPPED", "Metadata JSON too large to cache") + try: + logger.warning( + "Metadata cache skipped (payload too large) for %s (bytes=%s > max=%s)", + filepath, + int(nbytes), + int(max_bytes), + ) + except Exception: + pass + return Result.Err( + "CACHE_SKIPPED", + "Metadata JSON too large to cache", + original_bytes=int(nbytes), + max_bytes=int(max_bytes), + ) metadata_hash = MetadataHelpers.compute_metadata_hash(metadata_raw) - return await db.aexecute( + write_res = await db.aexecute( """ INSERT INTO metadata_cache (filepath, state_hash, metadata_hash, metadata_raw) @@ -478,6 +609,11 @@ class MetadataHelpers: """, (filepath, state_hash, metadata_hash, metadata_raw) ) + try: + await MetadataHelpers._maybe_cleanup_metadata_cache(db) + except Exception: + pass + return write_res @staticmethod def compute_metadata_hash(raw_json: str) -> str: diff --git a/backend/features/index/scanner.py b/backend/features/index/scanner.py index 1fecd37e..920b61c4 100644 --- a/backend/features/index/scanner.py +++ b/backend/features/index/scanner.py @@ -25,6 +25,7 @@ from ...config import ( SCAN_BATCH_LARGE, SCAN_BATCH_XL, MAX_TO_ENRICH_ITEMS, + IS_WINDOWS, ) from ..metadata import MetadataService from .metadata_helpers import MetadataHelpers @@ -58,7 +59,13 @@ class IndexScanner: and integration with metadata extraction. """ - def __init__(self, db: Sqlite, metadata_service: MetadataService, scan_lock: asyncio.Lock): + def __init__( + self, + db: Sqlite, + metadata_service: MetadataService, + scan_lock: asyncio.Lock, + index_lock: Optional[asyncio.Lock] = None, + ): """ Initialize index scanner. @@ -70,6 +77,7 @@ class IndexScanner: self.db = db self.metadata = metadata_service self._scan_lock = scan_lock + self._index_lock = index_lock or scan_lock self._current_scan_id = None async def scan_directory( @@ -217,6 +225,7 @@ class IndexScanner: to_enrich=to_enrich, ) batch = [] + await asyncio.sleep(0) if batch: await self._scan_stream_batch( @@ -230,6 +239,7 @@ class IndexScanner: to_enrich=to_enrich, ) batch = [] + await asyncio.sleep(0) except asyncio.CancelledError: stop_event.set() raise @@ -255,6 +265,16 @@ class IndexScanner: skipped=stats["skipped"], errors=stats["errors"] ) + if duration > 0.5: + logger.debug( + "scan_directory timing: %.3fs (scanned=%s added=%s updated=%s skipped=%s errors=%s)", + duration, + stats.get("scanned"), + stats.get("added"), + stats.get("updated"), + stats.get("skipped"), + stats.get("errors"), + ) self._current_scan_id = None # Return to_enrich list for background enrichment @@ -334,7 +354,7 @@ class IndexScanner: Returns: Result with indexing statistics """ - async with self._scan_lock: + async with self._index_lock: # Filter unsupported files to prevent indexing internal files (DBs, etc.) filtered_paths = [] for p in paths: @@ -361,7 +381,7 @@ class IndexScanner: self._current_scan_id = scan_id scan_start = time.perf_counter() - # Reduce log noise for single-file indexing (common in watchers) + # Reduce log noise for single-file indexing (common in incremental updates) start_log_level = logging.DEBUG if len(paths) == 1 else logging.INFO self._log_scan_event( start_log_level, @@ -379,7 +399,7 @@ class IndexScanner: "errors": 0, "start_time": datetime.now().isoformat() } - + added_ids: List[int] = [] try: for batch in self._chunk_file_batches(paths): if not batch: @@ -412,7 +432,9 @@ class IndexScanner: existing_map=existing_map, stats=stats, to_enrich=None, + added_ids=added_ids, ) + await asyncio.sleep(0) finally: stats["end_time"] = datetime.now().isoformat() duration = time.perf_counter() - scan_start @@ -438,6 +460,9 @@ class IndexScanner: ) self._current_scan_id = None + # Include added/updated asset IDs for WebSocket notification + if added_ids: + stats["added_ids"] = added_ids return Result.Ok(stats) async def _index_batch( @@ -452,6 +477,7 @@ class IndexScanner: existing_map: Dict[str, Dict[str, Any]], stats: Dict[str, Any], to_enrich: Optional[List[str]] = None, + added_ids: Optional[List[int]] = None, ) -> None: """ Index a batch of files using a single transaction for all DB writes. @@ -460,6 +486,8 @@ class IndexScanner: while keeping the "do not crash UI" and Result patterns intact. """ + batch_start = time.perf_counter() + # Phase 1: Stat files and determine which need metadata extraction prepared: List[Dict[str, Any]] = [] needs_metadata: List[tuple[Path, str, int, int, str, Optional[int]]] = [] # (path, filepath, mtime, size, state_hash, existing_id) @@ -655,7 +683,9 @@ class IndexScanner: # If the batch fails, fall back to processing items individually batch_failed = False try: - async with self.db.atransaction(mode="immediate"): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + raise RuntimeError(tx.error or "Failed to begin transaction") for entry in prepared: action = entry.get("action") @@ -726,6 +756,12 @@ class IndexScanner: int(entry.get("size") or 0), ) stats["updated"] += 1 + # Collect updated asset ID for WebSocket notification + if added_ids is not None and asset_id: + try: + added_ids.append(int(asset_id)) + except Exception: + pass if entry.get("fast") and to_enrich is not None: if len(to_enrich) < MAX_TO_ENRICH_ITEMS: to_enrich.append(entry.get("filepath") or str(entry.get("file_path") or "")) @@ -771,6 +807,12 @@ class IndexScanner: int(entry.get("size") or 0), ) stats["added"] += 1 + # Collect added asset ID for WebSocket notification + if added_ids is not None and res.data and res.data.get("asset_id"): + try: + added_ids.append(int(res.data["asset_id"])) + except Exception: + pass if entry.get("fast") and to_enrich is not None: if len(to_enrich) < MAX_TO_ENRICH_ITEMS: to_enrich.append(entry.get("filepath") or str(entry.get("file_path") or "")) @@ -780,29 +822,35 @@ class IndexScanner: # Unknown action: count as skipped to be safe. stats["skipped"] += 1 + if not tx.ok: + raise RuntimeError(tx.error or "Commit failed") except Exception as batch_error: # If the entire batch transaction fails, fall back to processing items individually batch_failed = True logger.warning("Batch transaction failed: %s. Falling back to individual processing.", str(batch_error)) stats["errors"] += len(prepared) # Temporarily count all as errors, will be corrected below + failed_entries: list[str] = [] # Process each item individually to isolate failures for entry in prepared: action = entry.get("action") + filepath_value = str(entry.get("filepath") or entry.get("file_path") or "unknown") if action in ("skipped", "skipped_journal"): stats["skipped"] += 1 - stats["errors"] -= 1 # Correct the error count + stats["errors"] = max(0, stats["errors"] - 1) # Correct the error count continue try: - async with self.db.atransaction(mode="immediate"): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + failed_entries.append(filepath_value) + continue if action == "refresh": asset_id = entry.get("asset_id") metadata_result = entry.get("metadata_result") if not asset_id or not isinstance(metadata_result, Result): - stats["skipped"] += 1 - stats["errors"] -= 1 # Correct the error count + failed_entries.append(filepath_value) continue try: refreshed = await MetadataHelpers.refresh_metadata_if_needed( @@ -817,12 +865,12 @@ class IndexScanner: self._write_scan_journal_entry, ) stats["skipped"] += 1 - stats["errors"] -= 1 # Correct the error count + stats["errors"] = max(0, stats["errors"] - 1) # Correct the error count if refreshed and entry.get("fast") and to_enrich is not None: if len(to_enrich) < MAX_TO_ENRICH_ITEMS: to_enrich.append(entry.get("filepath") or str(entry.get("file_path") or "")) except Exception as exc: - stats["errors"] += 1 # Keep as error + failed_entries.append(filepath_value) logger.warning("Metadata refresh failed for asset_id=%s: %s", asset_id, exc) continue @@ -830,7 +878,7 @@ class IndexScanner: asset_id = entry.get("asset_id") metadata_result = entry.get("metadata_result") if not asset_id or not isinstance(metadata_result, Result): - stats["errors"] += 1 # Keep as error + failed_entries.append(filepath_value) continue cache_store = bool(entry.get("cache_store")) if cache_store: @@ -862,17 +910,17 @@ class IndexScanner: int(entry.get("size") or 0), ) stats["updated"] += 1 - stats["errors"] -= 1 # Correct the error count + stats["errors"] = max(0, stats["errors"] - 1) # Correct the error count if entry.get("fast") and to_enrich is not None: to_enrich.append(entry.get("filepath") or str(entry.get("file_path") or "")) else: - stats["errors"] += 1 # Keep as error + failed_entries.append(filepath_value) continue if action == "added": metadata_result = entry.get("metadata_result") if not isinstance(metadata_result, Result): - stats["errors"] += 1 # Keep as error + failed_entries.append(filepath_value) continue cache_store = bool(entry.get("cache_store")) res = await self._add_asset( @@ -907,20 +955,34 @@ class IndexScanner: int(entry.get("size") or 0), ) stats["added"] += 1 - stats["errors"] -= 1 # Correct the error count + stats["errors"] = max(0, stats["errors"] - 1) # Correct the error count if entry.get("fast") and to_enrich is not None: to_enrich.append(entry.get("filepath") or str(entry.get("file_path") or "")) else: - stats["errors"] += 1 # Keep as error + failed_entries.append(filepath_value) continue # Unknown action: count as skipped to be safe. stats["skipped"] += 1 - stats["errors"] -= 1 # Correct the error count + stats["errors"] = max(0, stats["errors"] - 1) # Correct the error count + if not tx.ok: + failed_entries.append(filepath_value) except Exception as individual_error: # If individual processing also fails, log and keep as error - logger.warning("Individual processing failed for entry: %s. Error: %s", str(entry.get("filepath", "unknown")), str(individual_error)) + failed_entries.append(filepath_value) + logger.warning("Individual processing failed for entry: %s. Error: %s", filepath_value, str(individual_error)) # Error count is already correct + if failed_entries: + sample = failed_entries[:5] + logger.warning("Batch fallback completed with %s failures (sample: %s)", len(failed_entries), sample) + + duration = time.perf_counter() - batch_start + if duration > 0.2: + logger.debug( + "_index_batch timing: %.3fs (batch=%s)", + duration, + len(batch) if batch is not None else 0, + ) async def _prepare_index_entry( self, @@ -1245,6 +1307,13 @@ class IndexScanner: mtime = int(stat.st_mtime) size = stat.st_size + # Normalize Windows path casing to avoid duplicate rows differing only by case. + if IS_WINDOWS: + try: + file_path = Path(str(file_path)).resolve(strict=True) + except Exception: + pass + # Compute file fingerprint before any caching or journal logic (prevents use-before-set bugs) filepath = str(file_path) state_hash = self._compute_state_hash(filepath, mtime_ns, size) @@ -1266,12 +1335,22 @@ class IndexScanner: existing_asset = existing_state else: existing = await self.db.aquery( - "SELECT id, mtime FROM assets WHERE filepath = ?", + "SELECT id, mtime, filepath FROM assets WHERE filepath = ?", (filepath,) ) if existing.ok and existing.data and len(existing.data) > 0: existing_asset = existing.data[0] + elif IS_WINDOWS: + try: + existing_ci = await self.db.aquery( + "SELECT id, mtime, filepath FROM assets WHERE filepath = ? COLLATE NOCASE", + (filepath,) + ) + if existing_ci.ok and existing_ci.data and len(existing_ci.data) > 0: + existing_asset = existing_ci.data[0] + except Exception: + pass if existing_asset is not None: try: @@ -1286,7 +1365,10 @@ class IndexScanner: cached_metadata = await MetadataHelpers.retrieve_cached_metadata(self.db, filepath, state_hash) if cached_metadata and cached_metadata.ok: refreshed = False - async with self.db.atransaction(mode="immediate"): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + logger.warning("Metadata refresh skipped (transaction begin failed) for %s: %s", filepath, tx.error) + return Result.Ok({"action": "skipped"}) refreshed = await MetadataHelpers.refresh_metadata_if_needed( self.db, existing_id, @@ -1300,6 +1382,9 @@ class IndexScanner: ) if refreshed: await self._write_scan_journal_entry(filepath, base_dir, state_hash, mtime, size) + if not tx.ok: + logger.warning("Metadata refresh commit failed for %s: %s", filepath, tx.error) + return Result.Ok({"action": "skipped"}) action = "skipped_refresh" if refreshed else "skipped" return Result.Ok({"action": action}) @@ -1361,7 +1446,9 @@ class IndexScanner: if incremental and existing_mtime == mtime and existing_id: refreshed = False # Keep write transactions short: metadata extraction is already done. - async with self.db.atransaction(mode="immediate"): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") refreshed = await MetadataHelpers.refresh_metadata_if_needed( self.db, existing_id, @@ -1375,12 +1462,16 @@ class IndexScanner: ) if refreshed: await self._write_scan_journal_entry(filepath, base_dir, state_hash, mtime, size) + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Commit failed") action = "skipped_refresh" if refreshed else "skipped" return Result.Ok({"action": action}) # Update existing asset if existing_id: - async with self.db.atransaction(mode="immediate"): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") result = await self._update_asset( existing_id, file_path, @@ -1393,10 +1484,14 @@ class IndexScanner: ) if result.ok: await self._write_scan_journal_entry(filepath, base_dir, state_hash, mtime, size) + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Commit failed") return result # Add new asset - async with self.db.atransaction(mode="immediate"): + async with self.db.atransaction(mode="immediate") as tx: + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") result = await self._add_asset( filename, subfolder, @@ -1412,6 +1507,8 @@ class IndexScanner: ) if result.ok: await self._write_scan_journal_entry(filepath, base_dir, state_hash, mtime, size) + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Commit failed") return result async def _add_asset( @@ -1471,17 +1568,23 @@ class IndexScanner: if not asset_id: return Result.Err("INSERT_FAILED", "Failed to get inserted asset ID") if write_metadata: - metadata_write = await MetadataHelpers.write_asset_metadata_row(self.db, asset_id, metadata_result) - if not metadata_write.ok: - self._log_scan_event( - logging.WARNING, - "Failed to insert metadata row", - asset_id=asset_id, - error=metadata_write.error, - stage="metadata_write" + async with self.db.lock_for_asset(asset_id): + metadata_write = await MetadataHelpers.write_asset_metadata_row( + self.db, + asset_id, + metadata_result, + filepath=filepath, ) + if not metadata_write.ok: + self._log_scan_event( + logging.WARNING, + "Failed to insert metadata row", + asset_id=asset_id, + error=metadata_write.error, + stage="metadata_write" + ) - return Result.Ok({"action": "added"}) + return Result.Ok({"action": "added", "asset_id": asset_id}) async def _update_asset( self, @@ -1507,36 +1610,42 @@ class IndexScanner: height = meta.get("height") duration = meta.get("duration") - # Update assets table (NO workflow fields) - update_result = await self.db.aexecute( - """ - UPDATE assets - SET width = COALESCE(?, width), - height = COALESCE(?, height), - duration = COALESCE(?, duration), - size = ?, mtime = ?, - source = ?, root_id = ?, - indexed_at = CURRENT_TIMESTAMP - WHERE id = ? - """, - (width, height, duration, size, mtime, str(source or "output"), str(root_id) if root_id else None, asset_id) - ) + async with self.db.lock_for_asset(asset_id): + # Update assets table (NO workflow fields) + update_result = await self.db.aexecute( + """ + UPDATE assets + SET width = COALESCE(?, width), + height = COALESCE(?, height), + duration = COALESCE(?, duration), + size = ?, mtime = ?, + source = ?, root_id = ?, + indexed_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (width, height, duration, size, mtime, str(source or "output"), str(root_id) if root_id else None, asset_id) + ) - if not update_result.ok: - return Result.Err("UPDATE_FAILED", update_result.error) + if not update_result.ok: + return Result.Err("UPDATE_FAILED", update_result.error) - if write_metadata: - metadata_write = await MetadataHelpers.write_asset_metadata_row(self.db, asset_id, metadata_result) - if not metadata_write.ok: - self._log_scan_event( - logging.WARNING, - "Failed to update metadata row", - asset_id=asset_id, - error=metadata_write.error, - stage="metadata_write" + if write_metadata: + metadata_write = await MetadataHelpers.write_asset_metadata_row( + self.db, + asset_id, + metadata_result, + filepath=str(file_path) if file_path else None, ) + if not metadata_write.ok: + self._log_scan_event( + logging.WARNING, + "Failed to update metadata row", + asset_id=asset_id, + error=metadata_write.error, + stage="metadata_write" + ) - return Result.Ok({"action": "updated"}) + return Result.Ok({"action": "updated", "asset_id": asset_id}) def _scan_context(self, **kwargs) -> Dict[str, Any]: context = {"scan_id": self._current_scan_id} if self._current_scan_id else {} diff --git a/backend/features/index/searcher.py b/backend/features/index/searcher.py index 35d6b021..e472d39d 100644 --- a/backend/features/index/searcher.py +++ b/backend/features/index/searcher.py @@ -176,8 +176,8 @@ class IndexSearcher: a.width, a.height, a.duration, a.size, a.mtime, COALESCE(m.rating, 0) as rating, COALESCE(m.tags, '[]') as tags, - {metadata_tags_text_clause} COALESCE(m.has_workflow, 0) as has_workflow, - COALESCE(m.has_generation_data, 0) as has_generation_data, + {metadata_tags_text_clause} m.has_workflow as has_workflow, + m.has_generation_data as has_generation_data, json_extract(m.metadata_raw, '$.generation_time_ms') as generation_time_ms, json_extract(m.metadata_raw, '$.file_info.ctime') as file_creation_time, json_extract(m.metadata_raw, '$.file_info.birthtime') as file_birth_time @@ -249,8 +249,8 @@ class IndexSearcher: a.width, a.height, a.duration, a.size, a.mtime, COALESCE(m.rating, 0) as rating, COALESCE(m.tags, '[]') as tags, - {metadata_tags_text_clause} COALESCE(m.has_workflow, 0) as has_workflow, - COALESCE(m.has_generation_data, 0) as has_generation_data, + {metadata_tags_text_clause} m.has_workflow as has_workflow, + m.has_generation_data as has_generation_data, json_extract(m.metadata_raw, '$.generation_time_ms') as generation_time_ms, json_extract(m.metadata_raw, '$.file_info.ctime') as file_creation_time, json_extract(m.metadata_raw, '$.file_info.birthtime') as file_birth_time, @@ -405,8 +405,8 @@ class IndexSearcher: a.width, a.height, a.duration, a.size, a.mtime, COALESCE(m.rating, 0) as rating, COALESCE(m.tags, '[]') as tags, -{metadata_tags_text_clause} COALESCE(m.has_workflow, 0) as has_workflow, - COALESCE(m.has_generation_data, 0) as has_generation_data, +{metadata_tags_text_clause} m.has_workflow as has_workflow, + m.has_generation_data as has_generation_data, json_extract(m.metadata_raw, '$.generation_time_ms') as generation_time_ms, json_extract(m.metadata_raw, '$.file_info.ctime') as file_creation_time, json_extract(m.metadata_raw, '$.file_info.birthtime') as file_birth_time @@ -475,8 +475,8 @@ class IndexSearcher: a.width, a.height, a.duration, a.size, a.mtime, COALESCE(m.rating, 0) as rating, COALESCE(m.tags, '[]') as tags, -{metadata_tags_text_clause} COALESCE(m.has_workflow, 0) as has_workflow, - COALESCE(m.has_generation_data, 0) as has_generation_data, +{metadata_tags_text_clause} m.has_workflow as has_workflow, + m.has_generation_data as has_generation_data, json_extract(m.metadata_raw, '$.generation_time_ms') as generation_time_ms, json_extract(m.metadata_raw, '$.file_info.ctime') as file_creation_time, json_extract(m.metadata_raw, '$.file_info.birthtime') as file_birth_time, best.rank as rank @@ -690,8 +690,8 @@ class IndexSearcher: COALESCE(m.tags, '') AS tags, COALESCE(m.tags_text, '') AS tags_text, m.workflow_hash, - COALESCE(m.has_workflow, 0) AS has_workflow, - COALESCE(m.has_generation_data, 0) AS has_generation_data, + m.has_workflow AS has_workflow, + m.has_generation_data AS has_generation_data, json_extract(m.metadata_raw, '$.generation_time_ms') as generation_time_ms, COALESCE(m.metadata_raw, '{}') AS metadata_raw FROM assets a @@ -743,8 +743,8 @@ class IndexSearcher: COALESCE(m.tags, '') AS tags, COALESCE(m.tags_text, '') AS tags_text, m.workflow_hash, - COALESCE(m.has_workflow, 0) AS has_workflow, - COALESCE(m.has_generation_data, 0) AS has_generation_data, + m.has_workflow AS has_workflow, + m.has_generation_data AS has_generation_data, COALESCE(m.metadata_quality, 'none') AS metadata_quality, COALESCE(m.metadata_raw, '{{}}') AS metadata_raw FROM assets a @@ -829,8 +829,8 @@ class IndexSearcher: a.root_id, COALESCE(m.rating, 0) as rating, COALESCE(m.tags, '[]') as tags, - COALESCE(m.has_workflow, 0) as has_workflow, - COALESCE(m.has_generation_data, 0) as has_generation_data + m.has_workflow as has_workflow, + m.has_generation_data as has_generation_data FROM assets a LEFT JOIN asset_metadata m ON a.id = m.asset_id WHERE {IN_CLAUSE} diff --git a/backend/features/index/service.py b/backend/features/index/service.py index 18f5ac50..a3ab16cb 100644 --- a/backend/features/index/service.py +++ b/backend/features/index/service.py @@ -58,7 +58,6 @@ class IndexService: self._enricher = MetadataEnricher( db, metadata_service, - self._scan_lock, self._scanner._compute_state_hash, MetadataHelpers.prepare_metadata_fields, MetadataHelpers.metadata_error_payload, diff --git a/backend/features/index/updater.py b/backend/features/index/updater.py index db197d6f..3c94ead8 100644 --- a/backend/features/index/updater.py +++ b/backend/features/index/updater.py @@ -50,16 +50,17 @@ class AssetUpdater: if not check_result.ok or not check_result.data or len(check_result.data) == 0: return Result.Err("NOT_FOUND", f"Asset not found: {asset_id}") - # Update or insert asset_metadata - result = await self.db.aexecute( - """ - INSERT INTO asset_metadata (asset_id, rating, tags) - VALUES (?, ?, '[]') - ON CONFLICT(asset_id) DO UPDATE SET - rating = excluded.rating - """, - (asset_id, rating) - ) + # Update or insert asset_metadata (serialize per-asset to avoid race with enrichers) + async with self.db.lock_for_asset(asset_id): + result = await self.db.aexecute( + """ + INSERT INTO asset_metadata (asset_id, rating, tags) + VALUES (?, ?, '[]') + ON CONFLICT(asset_id) DO UPDATE SET + rating = excluded.rating + """, + (asset_id, rating) + ) if not result.ok: return Result.Err("UPDATE_FAILED", result.error) @@ -101,28 +102,29 @@ class AssetUpdater: # Build tags_text for FTS5 (if column exists) tags_text = " ".join(sanitized) if self._has_tags_text_column else None - # Update or insert asset_metadata - if self._has_tags_text_column: - result = await self.db.aexecute( - """ - INSERT INTO asset_metadata (asset_id, rating, tags, tags_text) - VALUES (?, 0, ?, ?) - ON CONFLICT(asset_id) DO UPDATE SET - tags = excluded.tags, - tags_text = excluded.tags_text - """, - (asset_id, tags_json, tags_text) - ) - else: - result = await self.db.aexecute( - """ - INSERT INTO asset_metadata (asset_id, rating, tags) - VALUES (?, 0, ?) - ON CONFLICT(asset_id) DO UPDATE SET - tags = excluded.tags - """, - (asset_id, tags_json) - ) + # Update or insert asset_metadata (serialize per-asset to avoid race with enrichers) + async with self.db.lock_for_asset(asset_id): + if self._has_tags_text_column: + result = await self.db.aexecute( + """ + INSERT INTO asset_metadata (asset_id, rating, tags, tags_text) + VALUES (?, 0, ?, ?) + ON CONFLICT(asset_id) DO UPDATE SET + tags = excluded.tags, + tags_text = excluded.tags_text + """, + (asset_id, tags_json, tags_text) + ) + else: + result = await self.db.aexecute( + """ + INSERT INTO asset_metadata (asset_id, rating, tags) + VALUES (?, 0, ?) + ON CONFLICT(asset_id) DO UPDATE SET + tags = excluded.tags + """, + (asset_id, tags_json) + ) if not result.ok: return Result.Err("UPDATE_FAILED", result.error) diff --git a/backend/features/index/watcher.py b/backend/features/index/watcher.py index e81af209..809e39c4 100644 --- a/backend/features/index/watcher.py +++ b/backend/features/index/watcher.py @@ -1,243 +1,478 @@ """ -Lightweight directory watcher that triggers incremental scans. -Replaced with Watchdog for real-time event handling. -""" -from __future__ import annotations +File system watcher for manual file additions/modifications. -import logging -import threading +Watches output and custom root directories for files added outside of ComfyUI +(e.g., manual copies, external tools). Does NOT conflict with entry.js which +handles ComfyUI executed events. +""" import asyncio +import os import time from pathlib import Path -from typing import Dict, Iterable, Optional +from threading import Lock +from typing import Dict, Set, Optional, Callable, Awaitable -from .service import IndexService -from ...shared import get_logger -from ...config import WATCHER_JOIN_TIMEOUT +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler, FileCreatedEvent, FileModifiedEvent, DirCreatedEvent + +from ...shared import get_logger, EXTENSIONS +from ...config import WATCHER_DEBOUNCE_MS, WATCHER_DEDUPE_TTL_MS logger = get_logger(__name__) +# Supported extensions (flattened) +SUPPORTED_EXTENSIONS: Set[str] = set() try: - from watchdog.observers import Observer - from watchdog.events import FileSystemEventHandler - HAS_WATCHDOG = True -except ImportError: - HAS_WATCHDOG = False - class Observer: - def schedule(self, *args, **kwargs): pass - def start(self): pass - def stop(self): pass - def join(self, **kwargs): pass - class FileSystemEventHandler: - pass - logger.warning("Watchdog module not found. Real-time file watching will be disabled.") + for exts in (EXTENSIONS or {}).values(): + for ext in (exts or []): + SUPPORTED_EXTENSIONS.add(str(ext).lower()) +except Exception: + SUPPORTED_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".gif", ".mp4", ".webm"} + +# Directories to always ignore (case-insensitive on Windows) +IGNORED_DIRS = { + "_mjr_index", + ".majoor", + "__pycache__", + ".git", + "node_modules", + ".thumbs", + ".cache", +} + +# Minimum file size (bytes) to avoid indexing partial/temp writes +MIN_FILE_SIZE = 100 +_RECENT_GENERATED_TTL_S = 30.0 +_RECENT_GENERATED_LOCK = Lock() +_RECENT_GENERATED: Dict[str, float] = {} +_MAX_PENDING_FILES = int(os.environ.get("MJR_WATCHER_PENDING_MAX", "5000") or 5000) -class IndexEventHandler(FileSystemEventHandler): - """Handles file system events and queues index updates.""" - - def __init__(self, watcher): - self.watcher = watcher - self.last_events: Dict[str, float] = {} - self.debounce_seconds = 1.0 +class DebouncedWatchHandler(FileSystemEventHandler): + """ + Handles file system events with deduplication and debouncing. - def _is_valid_file(self, path_str: str) -> bool: - try: - p = Path(path_str) - if p.name.startswith(".") or not p.name: - return False - - # Ignore SQLite database files and journals to prevent feedback loops - # when the DB is located within the watched directory. - name_lower = p.name.lower() - if (name_lower.endswith(".db") or - name_lower.endswith(".db-journal") or - name_lower.endswith(".db-shm") or - name_lower.endswith(".db-wal") or - name_lower.endswith(".sqlite") or - name_lower.endswith(".sqlite3") or - "-journal" in name_lower): - return False - - # Ignore common temp extensions and partial downloads - if p.suffix.lower() in {".tmp", ".crdownload", ".part", ".lock", ".aria2"}: - return False - return True - except Exception: - return False - - def _debounce(self, filepath: str) -> bool: - # Simple debounce to avoid double-processing (created + modified) - now = time.time() - last = self.last_events.get(filepath, 0) - if now - last < self.debounce_seconds: - return True - self.last_events[filepath] = now - # Cleanup old entries - if len(self.last_events) > 1000: - self.last_events.clear() - return False - - def on_created(self, event): - if event.is_directory or not self._is_valid_file(event.src_path): - return - if self._debounce(event.src_path): - return - # logger.debug(f"Watcher: Created {event.src_path}") - self.watcher.queue_update(event.src_path, action="add") - - def on_moved(self, event): - if event.is_directory: - return - - # REMOVE old path - if self._is_valid_file(event.src_path): - # logger.debug(f"Watcher: Moved from {event.src_path}") - self.watcher.queue_update(event.src_path, action="remove") - - # ADD new path - if self._is_valid_file(event.dest_path): - # logger.debug(f"Watcher: Moved to {event.dest_path}") - self.watcher.queue_update(event.dest_path, action="add") - - def on_deleted(self, event): - if event.is_directory: - return - # logger.debug(f"Watcher: Deleted {event.src_path}") - self.watcher.queue_update(event.src_path, action="remove") - - def on_modified(self, event): - if event.is_directory or not self._is_valid_file(event.src_path): - return - if self._debounce(event.src_path): - return - # logger.debug(f"Watcher: Modified {event.src_path}") - self.watcher.queue_update(event.src_path, action="add") - - -class DirectoryWatcher: - """Uses watchdog to listen for file events without full rescans.""" + - Ignores non-media files and internal directories + - Deduplicates rapid events for the same file + - Batches files before triggering indexing + - Only reacts to file creation (not modification noise) + """ def __init__( self, - index_service: IndexService, - directories: Iterable[str], - interval_seconds: Optional[float] = None, # Deprecated/Unused - join_timeout: Optional[float] = None + on_files_ready: Callable[[list], Awaitable[None]], + loop: asyncio.AbstractEventLoop, + debounce_ms: int = 500, + dedupe_ttl_ms: int = 3000, ): - self.index_service = index_service - self.directories = [Path(dir_path).resolve() for dir_path in directories if dir_path] - # NOTE: Do NOT cache the event loop here. The loop may not exist yet at __init__ time - # (e.g., when DirectoryWatcher is instantiated before asyncio.run()). Instead, we - # fetch the running loop dynamically in queue_update() and _setup_observer(). - self._join_timeout = join_timeout if join_timeout is not None else WATCHER_JOIN_TIMEOUT - self._observer = None - self._shutdown = False - - if not HAS_WATCHDOG: - logger.warning("File watcher disabled: Watchdog module missing. Manual refresh required.") + super().__init__() + self._on_files_ready = on_files_ready + self._loop = loop + self._debounce_s = debounce_ms / 1000.0 + self._dedupe_ttl_s = dedupe_ttl_ms / 1000.0 - def _setup_observer(self): - if not HAS_WATCHDOG: - return None - - observer = Observer() - handler = IndexEventHandler(self) - scheduled = 0 - for d in self.directories: - if d.exists() and d.is_dir(): - try: - observer.schedule(handler, str(d), recursive=True) - scheduled += 1 - except Exception as exc: - logger.warning(f"Failed to watch {d}: {exc}") - - if scheduled > 0: - logger.info(f"File watcher initialized for {scheduled} directories (Mode: Event-Driven/Watchdog)") - return observer + self._lock = Lock() + self._pending: Dict[str, float] = {} # filepath -> timestamp + self._recent: Dict[str, float] = {} # filepath -> last processed timestamp + self._flush_timer: Optional[asyncio.TimerHandle] = None - def start(self): - """Start the background watcher thread.""" - if self._observer and self._observer.is_alive(): + def _is_ignored_path(self, path: str) -> bool: + """Check if path is inside an ignored directory.""" + try: + parts = Path(path).parts + for part in parts: + if part.lower() in IGNORED_DIRS or part.startswith("."): + return True + except Exception: + pass + return False + + def _is_supported(self, path: str) -> bool: + """Check if file has a supported extension.""" + try: + ext = Path(path).suffix.lower() + return ext in SUPPORTED_EXTENSIONS + except Exception: + return False + + def _normalize_path(self, path: str) -> str: + """Normalize path for deduplication.""" + try: + return os.path.normcase(os.path.normpath(path)) + except Exception: + return path + + def on_created(self, event): + if isinstance(event, DirCreatedEvent): + return + if not isinstance(event, FileCreatedEvent): + return + self._handle_file(event.src_path) + + def on_modified(self, event): + # Ignore modifications to avoid re-indexing existing files. + # We only care about new files added manually. + return + + def _handle_file(self, path: str): + """Queue a file for indexing with deduplication.""" + if not path: return - self._observer = self._setup_observer() - if self._observer: - try: - self._observer.start() - logger.info("File watcher started (event-based)") - except Exception as exc: - logger.error(f"Failed to start file watcher: {exc}") - - def stop(self): - """Request the watcher thread to stop.""" - self._shutdown = True - if self._observer: - try: - self._observer.stop() - self._observer.join(timeout=self._join_timeout) - except Exception: - pass - logger.info("File watcher stopped") - - def is_alive(self) -> bool: - """Return True if the watcher thread is running.""" - return self._observer.is_alive() if self._observer else False - - def queue_update(self, filepath: str, action: str): - if self._shutdown: + # Fast rejections + if self._is_ignored_path(path): return - - # Capture values needed in the coroutine (avoid closure issues) - _filepath = str(filepath) - _action = str(action) - _directories = list(self.directories) - _index_service = self.index_service - - async def _process(): + if not self._is_supported(path): + return + # NOTE: _is_recent_generated is checked at flush time (not here) to avoid + # a race condition where the watcher detects a file before the frontend + # executed-event handler has called mark_recent_generated(). + + key = self._normalize_path(path) + now = time.time() + + with self._lock: + # Skip if recently processed + last = self._recent.get(key, 0) + if last and (now - last) < self._dedupe_ttl_s: + return + + # Prevent unbounded memory growth under bursts. + if _MAX_PENDING_FILES > 0 and len(self._pending) >= _MAX_PENDING_FILES: + return + + # Add to pending batch + self._pending[key] = now + + # Schedule flush + self._schedule_flush() + + def _schedule_flush(self): + """Schedule a debounced flush of pending files.""" + try: + if self._flush_timer: + self._flush_timer.cancel() + self._flush_timer = self._loop.call_later( + self._debounce_s, + lambda: asyncio.run_coroutine_threadsafe(self._flush(), self._loop) + ) + except Exception: + pass + + async def _flush(self): + """Flush pending files to the indexer.""" + with self._lock: + if not self._pending: + return + + now = time.time() + candidates = list(self._pending.keys()) + self._pending.clear() + self._flush_timer = None + + # Filter out files that are too small (still being written), vanished, + # or recently indexed via the frontend executed-event path. + files = [] + for f in candidates: + if _is_recent_generated(f): + continue try: - # Small delay to allow file writes to settle (e.g. copy paste) - if _action == "add": - await asyncio.sleep(0.5) - - if _action == "remove": - await _index_service.remove_file(_filepath) - elif _action == "add": - # Determine base dir for relative path calculation - path_obj = Path(_filepath) - base_dir = str(path_obj.parent) - for d in _directories: - if str(d) in _filepath: - base_dir = str(d) - break - - await _index_service.index_paths( - paths=[path_obj], - base_dir=base_dir, - incremental=True, - source="output" # Default to output, logic could be refined - ) - except Exception as exc: - logger.debug(f"Watcher update processing failed for {_filepath}: {exc}") + size = os.path.getsize(f) + if size >= MIN_FILE_SIZE: + files.append(f) + except OSError: + pass # File vanished or inaccessible + + if not files: + return + + # Mark as recently processed + with self._lock: + now = time.time() + for f in files: + self._recent[f] = now + + # Prune old entries from recent + to_remove = [k for k, v in self._recent.items() if (now - v) > self._dedupe_ttl_s * 2] + for k in to_remove: + self._recent.pop(k, None) try: - # Get the running loop dynamically (may have changed since __init__) - try: - loop = asyncio.get_running_loop() - except RuntimeError: - # No running loop - try to get the default loop (may not exist) + await self._on_files_ready(files) + except Exception as e: + logger.debug("Watcher flush error: %s", e) + + +def _normalize_recent_key(path: str) -> str: + try: + return os.path.normcase(os.path.normpath(path)) + except Exception: + return str(path or "") + + +def _is_recent_generated(path: str) -> bool: + key = _normalize_recent_key(path) + if not key: + return False + now = time.time() + try: + with _RECENT_GENERATED_LOCK: + if _RECENT_GENERATED: + to_remove = [k for k, ts in _RECENT_GENERATED.items() if (now - float(ts or 0)) > _RECENT_GENERATED_TTL_S] + for k in to_remove: + _RECENT_GENERATED.pop(k, None) + ts = _RECENT_GENERATED.get(key) + if ts and (now - float(ts or 0)) <= _RECENT_GENERATED_TTL_S: + return True + except Exception: + return False + return False + + +def is_recent_generated(path: str) -> bool: + return _is_recent_generated(path) + + +def mark_recent_generated(paths: list[str]) -> None: + if not paths: + return + now = time.time() + try: + with _RECENT_GENERATED_LOCK: + for p in paths: + key = _normalize_recent_key(p) + if key: + _RECENT_GENERATED[key] = now + if len(_RECENT_GENERATED) > 5000: + # Drop oldest entries (best-effort) + for k, _ in sorted(_RECENT_GENERATED.items(), key=lambda kv: kv[1])[:1000]: + _RECENT_GENERATED.pop(k, None) + except Exception: + return + + +class OutputWatcher: + """ + Watches output and custom root directories for manual file additions. + + Usage: + watcher = OutputWatcher(index_callback) + await watcher.start([{"path": "/path/to/output", "source": "output"}]) + ... + await watcher.stop() + """ + + def __init__(self, index_callback: Callable[[list, str, Optional[str], Optional[str]], Awaitable[None]]): + """ + Args: + index_callback: async function(filepaths, base_dir, source, root_id) to index files + """ + self._index_callback = index_callback + self._observer: Optional[Observer] = None + self._handler: Optional[DebouncedWatchHandler] = None + self._watched_paths: Dict[str, dict] = {} # watch_key -> {path, source, root_id, watch} + self._lock = Lock() + self._running = False + self._allowed_sources: set[str] = set() + + async def start(self, paths: list, loop: Optional[asyncio.AbstractEventLoop] = None): + """Start watching the given directories.""" + if self._running: + return + + loop = loop or asyncio.get_event_loop() + self._allowed_sources = set() + + async def on_files_ready(files: list): + if not files: + return + # Group by watched root for proper base_dir + by_dir: Dict[str, dict] = {} + for f in files: try: - loop = asyncio.get_event_loop() - if loop.is_closed(): - logger.debug("Event loop is closed, cannot queue watcher update") + # Find the watched root this file belongs to + normalized_f = os.path.normcase(os.path.normpath(f)) + best = None + best_len = -1 + for entry in self._watched_paths.values(): + watched = entry.get("path") + if not watched: + continue + normalized_watched = os.path.normcase(os.path.normpath(watched)) + if _is_under_path(normalized_f, normalized_watched) and len(normalized_watched) > best_len: + best = entry + best_len = len(normalized_watched) + + if not best: + continue + + base = best.get("path") + if not base: + continue + key = str(base) + if key not in by_dir: + by_dir[key] = { + "files": [], + "source": best.get("source"), + "root_id": best.get("root_id"), + } + by_dir[key]["files"].append(f) + except Exception: + pass + + for base_dir, payload in by_dir.items(): + try: + await self._index_callback( + payload.get("files") or [], + base_dir, + payload.get("source"), + payload.get("root_id"), + ) + except Exception as e: + logger.debug("Watcher index error: %s", e) + + self._handler = DebouncedWatchHandler( + on_files_ready, + loop, + debounce_ms=WATCHER_DEBOUNCE_MS, + dedupe_ttl_ms=WATCHER_DEDUPE_TTL_MS, + ) + self._observer = Observer() + + for path in paths: + try: + if isinstance(path, dict): + raw_path = path.get("path") + source = path.get("source") + root_id = path.get("root_id") + else: + raw_path = path + source = None + root_id = None + + if not raw_path: + continue + normalized = os.path.normpath(raw_path) + if os.path.isdir(normalized): + watch = self._observer.schedule(self._handler, normalized, recursive=True) + source_norm = str(source or "").strip().lower() if source is not None else None + if source_norm: + self._allowed_sources.add(source_norm) + self._watched_paths[str(id(watch))] = { + "path": normalized, + "source": source_norm, + "root_id": root_id, + "watch": watch, + } + logger.info("Watcher started for: %s", normalized) + except Exception as e: + logger.warning("Failed to watch %s: %s", path, e) + + if self._watched_paths: + self._observer.start() + self._running = True + logger.info("File watcher started for %d directories", len(self._watched_paths)) + + async def stop(self): + """Stop watching all directories.""" + if not self._running: + return + + try: + if self._observer: + self._observer.stop() + self._observer.join(timeout=2) + self._observer = None + except Exception as e: + logger.debug("Watcher stop error: %s", e) + + self._watched_paths.clear() + self._handler = None + self._running = False + logger.info("File watcher stopped") + + def add_path(self, path: str, *, source: Optional[str] = None, root_id: Optional[str] = None): + """Add a new path to watch (e.g., when custom root is added).""" + if not self._running or not self._observer or not self._handler: + return + + try: + normalized = os.path.normpath(path) + if os.path.isdir(normalized): + # Check if already watched + for entry in self._watched_paths.values(): + if entry.get("path") == normalized: return - except RuntimeError: - logger.debug("No event loop available for watcher update") + source_norm = str(source or "").strip().lower() if source is not None else None + if not self._allows_source(source_norm): + try: + logger.debug("Watcher ignored path (scope mismatch): %s (source=%s)", normalized, source_norm) + except Exception: + pass return - - asyncio.run_coroutine_threadsafe(_process(), loop) - except Exception as exc: - logger.debug(f"Failed to queue watcher update: {exc}") + watch = self._observer.schedule(self._handler, normalized, recursive=True) + self._watched_paths[str(id(watch))] = { + "path": normalized, + "source": source_norm, + "root_id": root_id, + "watch": watch, + } + logger.info("Watcher added: %s", normalized) + except Exception as e: + logger.warning("Failed to add watch for %s: %s", path, e) -# Legacy implementation removed. Watchdog is now required for auto-updates without full scan. + def remove_path(self, path: str): + """Remove a path from watching (e.g., when custom root is removed).""" + if not self._running or not self._observer: + return + try: + normalized = os.path.normpath(path) + # Find and remove the watch + to_remove = None + for key, entry in self._watched_paths.items(): + if entry.get("path") == normalized: + to_remove = key + break + + if to_remove: + entry = self._watched_paths.pop(to_remove, None) or {} + try: + if self._observer and entry.get("watch"): + self._observer.unschedule(entry.get("watch")) + except Exception: + pass + logger.info("Watcher removed: %s", normalized) + except Exception as e: + logger.debug("Failed to remove watch for %s: %s", path, e) + + @property + def is_running(self) -> bool: + return self._running + + @property + def watched_directories(self) -> list: + return [entry.get("path") for entry in self._watched_paths.values() if entry.get("path")] + + def _allows_source(self, source: Optional[str]) -> bool: + if not self._allowed_sources: + return True + if not source: + return False + return str(source).strip().lower() in self._allowed_sources + + +def _is_under_path(candidate: str, root: str) -> bool: + if not candidate or not root: + return False + try: + common = os.path.commonpath([candidate, root]) + if common == root: + return True + except Exception: + pass + try: + root_sep = root if root.endswith(os.sep) else root + os.sep + return candidate.startswith(root_sep) or candidate == root + except Exception: + return False diff --git a/backend/features/index/watcher_scope.py b/backend/features/index/watcher_scope.py new file mode 100644 index 00000000..b7018566 --- /dev/null +++ b/backend/features/index/watcher_scope.py @@ -0,0 +1,107 @@ +""" +Watcher scope persistence and watch path resolution. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Optional + +try: + import folder_paths # type: ignore +except Exception: + class _FolderPathsStub: + @staticmethod + def get_input_directory() -> str: + return str((Path(__file__).resolve().parents[4] / "input").resolve()) + + folder_paths = _FolderPathsStub() # type: ignore + +from ...config import OUTPUT_ROOT +from ...custom_roots import resolve_custom_root, list_custom_roots + +WATCHER_SCOPE_KEY = "watcher_scope" +WATCHER_CUSTOM_ROOT_ID_KEY = "watcher_custom_root_id" + + +def normalize_scope(scope: Optional[str]) -> str: + s = str(scope or "").strip().lower() or "output" + if s in ("outputs",): + return "output" + if s in ("inputs",): + return "input" + allowed = {"output", "input", "all", "custom"} + return s if s in allowed else "output" + + +def build_watch_paths(scope: str, custom_root_id: Optional[str]) -> list[dict]: + watch_paths: list[dict] = [] + s = normalize_scope(scope) + + if s == "output": + if OUTPUT_ROOT and os.path.isdir(OUTPUT_ROOT): + watch_paths.append({"path": OUTPUT_ROOT, "source": "output", "root_id": None}) + return watch_paths + + if s == "input": + try: + input_root = folder_paths.get_input_directory() + except Exception: + input_root = None + if input_root and os.path.isdir(input_root): + watch_paths.append({"path": input_root, "source": "input", "root_id": None}) + return watch_paths + + if s == "custom": + root_result = resolve_custom_root(str(custom_root_id or "")) + if root_result.ok: + path = str(root_result.data) + if path and os.path.isdir(path): + watch_paths.append({"path": path, "source": "custom", "root_id": str(custom_root_id or "")}) + return watch_paths + + if s == "all": + if OUTPUT_ROOT and os.path.isdir(OUTPUT_ROOT): + watch_paths.append({"path": OUTPUT_ROOT, "source": "output", "root_id": None}) + try: + input_root = folder_paths.get_input_directory() + if input_root and os.path.isdir(input_root): + watch_paths.append({"path": input_root, "source": "input", "root_id": None}) + except Exception: + pass + try: + roots_result = list_custom_roots() + if roots_result.ok: + for root in (roots_result.data or []): + path = root.get("path") + if path and os.path.isdir(path): + watch_paths.append({"path": path, "source": "custom", "root_id": root.get("id")}) + except Exception: + pass + return watch_paths + + return watch_paths + + +async def load_watcher_scope(db) -> dict: + scope = "output" + custom_root_id = "" + try: + res = await db.aquery( + "SELECT key, value FROM metadata WHERE key IN (?, ?)", + (WATCHER_SCOPE_KEY, WATCHER_CUSTOM_ROOT_ID_KEY), + ) + if res.ok and res.data: + for row in res.data: + key = str(row.get("key") or "") + val = str(row.get("value") or "") + if key == WATCHER_SCOPE_KEY and val: + scope = val + elif key == WATCHER_CUSTOM_ROOT_ID_KEY: + custom_root_id = val + except Exception: + pass + return { + "scope": normalize_scope(scope), + "custom_root_id": str(custom_root_id or "").strip(), + } diff --git a/backend/features/metadata/archived/native_extractors.py b/backend/features/metadata/archived/native_extractors.py deleted file mode 100644 index 7e12cc2a..00000000 --- a/backend/features/metadata/archived/native_extractors.py +++ /dev/null @@ -1,292 +0,0 @@ -""" -Native metadata extractors using Python libraries (Pillow) instead of external tools (ExifTool). -Much faster for reading ComfyUI metadata (Step 1). -""" -import asyncio -import json -import logging -from typing import Optional, Dict, Any, List -from pathlib import Path - -from ...shared import Result, ErrorCode -from .parsing_utils import ( - parse_json_value, - looks_like_comfyui_workflow, - looks_like_comfyui_prompt_graph, - try_parse_json_text, - parse_auto1111_params -) - -try: - from PIL import Image, ExifTags - from PIL.PngImagePlugin import PngImageFile - HAS_PIL = True -except ImportError: - HAS_PIL = False - -logger = logging.getLogger(__name__) - -def _extract_fallback_from_workflow_nodes(nodes: list) -> Dict[str, Any]: - """ - Fallback heuristic to extract geninfo (seed, prompts) directly from workflow UI graph - when the execution 'prompt' graph is missing. - """ - geninfo: Dict[str, Any] = {} - if not nodes: - return geninfo - - nodes_by_id = {n['id']: n for n in nodes} - link_map = {} # link_id -> node_id (source) - - for n in nodes: - for out in n.get('outputs', []): - if out.get('links'): - for link_id in out['links']: - link_map[str(link_id)] = n['id'] - - def get_source_node(link_id): - nid = link_map.get(str(link_id)) - return nodes_by_id.get(nid) - - positives = set() - negatives = set() - found_seed = None - - def trace_text(node, depth=0): - if not node: return None - if depth > 5: return None - - ntype = node.get('type', '') - # Text Node - if ('TextEncode' in ntype or 'Prompt' in ntype or 'PrimitiveString' in ntype) and 'Note' not in ntype: - vals = node.get('widgets_values') - if vals and isinstance(vals[0], str) and len(vals[0].strip()) > 0: - return vals[0] - - # Recurse upstream - inputs = node.get('inputs', []) - for inp in inputs: - name = inp.get('name', '').lower() - if name in ('positive', 'negative', 'conditioning', 'guider', 'text', 'string'): - link = inp.get('link') - if link: - src = get_source_node(link) - res = trace_text(src, depth+1) - if res: return res - return None - - # 1. Identify Samplers to find Pos/Neg roots - samplers = [n for n in nodes if "Sampler" in n.get('type', '')] - for s in samplers: - for inp in s.get("inputs", []): - name = inp.get("name", "").lower() - link = inp.get("link") - if not link: continue - - if name == "positive": - txt = trace_text(get_source_node(link)) - if txt: positives.add(txt) - elif name == "negative": - txt = trace_text(get_source_node(link)) - if txt: negatives.add(txt) - elif name == "guider": - # Flux/Guider inputs - gnode = get_source_node(link) - if gnode: - for ginp in gnode.get("inputs", []): - gname = ginp.get("name", "").lower() - glink = ginp.get("link") - if gname == "positive": - txt = trace_text(get_source_node(glink)) - if txt: positives.add(txt) - elif gname == "negative": - txt = trace_text(get_source_node(glink)) - if txt: negatives.add(txt) - - # 2. General scan for seeds (and prompts if graph tracing failed) - found_prompts_fallback = set() - for n in nodes: - ntype = n.get('type', '') - vals = n.get('widgets_values', []) - - # Seeds - if 'Seed' in ntype or 'Noise' in ntype or 'Sampler' in ntype: - for v in vals: - if isinstance(v, (int, float)): - iv = int(v) - if iv > 1000000 and found_seed is None: - found_seed = iv - - # Fallback prompts if structured trace found nothing - if not positives and not negatives: - if ('TextEncode' in ntype or 'Prompt' in ntype) and 'Note' not in ntype: - for v in vals: - if isinstance(v, str) and len(v.strip()) > 0: - found_prompts_fallback.add(v) - - if positives: - geninfo["positive"] = list(positives) - if negatives: - geninfo["negative"] = list(negatives) - - if not positives and found_prompts_fallback: - geninfo["positive"] = list(found_prompts_fallback) - - if found_seed: - geninfo["seed"] = found_seed - - return geninfo - -def _scan_text_values_for_metadata(values: List[str]) -> Dict[str, Any]: - """ - Scans a list of text values (from EXIF, PNG chunks, etc) for - ComfyUI workflow/prompt and A1111 parameters. - """ - found: Dict[str, Any] = {} - workflow = None - prompt = None - - for val in values: - if not val or not isinstance(val, str): - continue - - # 1. Try to parse as JSON (handles prefixes) - parsed = parse_json_value(val) - - # 2. Check wrappers { "workflow": ..., "prompt": ... } - if isinstance(parsed, dict): - wf_wrap = parsed.get("workflow") or parsed.get("Workflow") - pr_wrap = parsed.get("prompt") or parsed.get("Prompt") - - if wf_wrap and isinstance(wf_wrap, dict): - if looks_like_comfyui_workflow(wf_wrap): - if not workflow: workflow = wf_wrap - - if pr_wrap: - if isinstance(pr_wrap, dict) and looks_like_comfyui_prompt_graph(pr_wrap): - if not prompt: prompt = pr_wrap - elif isinstance(pr_wrap, str): - pr_parsed = try_parse_json_text(pr_wrap) - if pr_parsed and looks_like_comfyui_prompt_graph(pr_parsed): - if not prompt: prompt = pr_parsed - - # 3. Direct Match (the value itself is the workflow/prompt) - if looks_like_comfyui_workflow(parsed): - if not workflow: workflow = parsed - elif looks_like_comfyui_prompt_graph(parsed): - if not prompt: prompt = parsed - - # 4. A1111 Fallback / Parameters - if "parameters" not in found: - params = parse_auto1111_params(val) - if params: - found["parameters"] = val - found.update(params) - - if workflow: - found["workflow"] = workflow - if prompt: - found["prompt"] = prompt - - return found - -def _extract_image_metadata_native_sync(path: str) -> Optional[Dict[str, Any]]: - """ - Sync worker for extracting image metadata using Pillow. - """ - if not HAS_PIL: - return None - - try: - with Image.open(path) as img: - info = img.info or {} - - # Dimensions (always available in PIL for valid images) - width, height = img.size - result = { - "width": width, - "height": height - } - - # --- Format-Generic Text Scan --- - candidate_texts = [] - - # PNG: Gather all text chunks - if img.format == "PNG": - for k, v in info.items(): - if isinstance(v, str): - candidate_texts.append(v) - elif isinstance(v, bytes): - try: - candidate_texts.append(v.decode('utf-8', errors='ignore')) - except Exception: - pass - - # WEBP: Gather EXIF / XMP / Info - elif img.format == "WEBP": - # Info dictionary (sometimes has keys) - for k, v in info.items(): - if isinstance(v, str): - candidate_texts.append(v) - elif isinstance(v, bytes): - try: - candidate_texts.append(v.decode('utf-8', errors='ignore')) - except Exception: - pass - - # EXIF Tags - try: - exif = img.getexif() - if exif: - # 270: ImageDescription, 271: Make, 272: Model, 305: Software, 37510: UserComment - candidate_tags = [270, 271, 272, 305, 37510, 0x9286] - for tag_id in candidate_tags: - val = exif.get(tag_id) - if val: - if isinstance(val, bytes): - try: - val = val.decode('utf-8', errors='ignore').strip('\x00') - except Exception: - continue - if isinstance(val, str) and len(val) > 2: - candidate_texts.append(val) - except Exception: - pass - - # --- Perform Scan --- - if candidate_texts: - scan_result = _scan_text_values_for_metadata(candidate_texts) - result.update(scan_result) - - # --- Fallback: GenInfo from Workflow --- - # We want to run this if we don't have a PROMPT GRAPH (dict). - # If "prompt" is just a string (A1111 text), we still might get better info from the workflow. - has_prompt_graph = isinstance(result.get("prompt"), dict) - - if not has_prompt_graph and "workflow" in result: - try: - nodes = result["workflow"].get("nodes") - if nodes: - fallback_geninfo = _extract_fallback_from_workflow_nodes(nodes) - if fallback_geninfo: - result["geninfo"] = fallback_geninfo - except Exception as e: - logger.debug(f"Workflow fallback extraction failed for {path}: {e}") - - return result - - except Exception as e: - logger.debug(f"Native extraction failed for {path}: {e}") - - return None - -async def extract_image_metadata_native(path: str) -> Optional[Dict[str, Any]]: - """ - Fast async extraction of image metadata (dimensions + ComfyUI/A1111 tags) using Pillow. - Offloads blocking IO to a thread. - Returns dict if found, None if failed. - """ - return await asyncio.to_thread(_extract_image_metadata_native_sync, path) - -# Legacy alias -extract_png_metadata_native = extract_image_metadata_native diff --git a/backend/features/metadata/service.py b/backend/features/metadata/service.py index 122eaf85..ba6feb36 100644 --- a/backend/features/metadata/service.py +++ b/backend/features/metadata/service.py @@ -10,6 +10,7 @@ from pathlib import Path from typing import Dict, Any, Optional from ...shared import Result, ErrorCode, get_logger, classify_file, log_structured +from ...config import METADATA_EXTRACT_CONCURRENCY from ...adapters.tools import ExifTool, FFProbe from ...settings import AppSettings from ...probe_router import pick_probe_backend @@ -156,6 +157,13 @@ class MetadataService: self.exiftool = exiftool self.ffprobe = ffprobe self._settings = settings + try: + max_concurrency = int(METADATA_EXTRACT_CONCURRENCY or 1) + except Exception: + max_concurrency = 1 + if max_concurrency <= 0: + max_concurrency = 1 + self._extract_sem = asyncio.Semaphore(max_concurrency) async def _enrich_with_geninfo_async(self, combined: Dict[str, Any]) -> None: """Helper to parse geninfo from prompt/workflow in combined metadata (Worker Thread).""" @@ -215,6 +223,15 @@ class MetadataService: if not os.path.exists(file_path): return Result.Err(ErrorCode.NOT_FOUND, f"File not found: {file_path}") + async with self._extract_sem: + return await self._get_metadata_impl(file_path, scan_id=scan_id, probe_mode_override=probe_mode_override) + + async def _get_metadata_impl( + self, + file_path: str, + scan_id: Optional[str] = None, + probe_mode_override: Optional[str] = None + ) -> Result[Dict[str, Any]]: # Get file kind kind = classify_file(file_path) if kind == "unknown": @@ -537,6 +554,22 @@ class MetadataService: if not file_paths: return {} + async with self._extract_sem: + return await self._get_metadata_batch_impl( + file_paths, + scan_id=scan_id, + probe_mode_override=probe_mode_override, + ) + + async def _get_metadata_batch_impl( + self, + file_paths: list[str], + scan_id: Optional[str] = None, + probe_mode_override: Optional[str] = None, + ) -> Dict[str, Result[Dict[str, Any]]]: + if not file_paths: + return {} + from ...shared import classify_file def _group_existing_paths(paths: list[str]): diff --git a/backend/routes/core/security.py b/backend/routes/core/security.py index 4a22a3cc..d7a6ab71 100644 --- a/backend/routes/core/security.py +++ b/backend/routes/core/security.py @@ -54,13 +54,13 @@ _rate_limit_lock = threading.Lock() _rate_limit_cleanup_counter = 0 -def _env_truthy(name: str) -> bool: +def _env_truthy(name: str, default: bool = False) -> bool: try: raw = os.environ.get(name) except Exception: raw = None if raw is None: - return False + return bool(default) return str(raw).strip().lower() in ("1", "true", "yes", "on") @@ -303,7 +303,8 @@ def _check_write_access(*, peer_ip: str, headers: Mapping[str, str]) -> "Result[ configured = _get_write_token() require_auth = _env_truthy("MAJOOR_REQUIRE_AUTH") - allow_remote = _env_truthy("MAJOOR_ALLOW_REMOTE_WRITE") + # By default, remote write access is NOT allowed unless explicitly configured. + allow_remote = _env_truthy("MAJOOR_ALLOW_REMOTE_WRITE", default=False) client_ip = _resolve_client_ip(peer_ip, headers) diff --git a/backend/routes/core/services.py b/backend/routes/core/services.py index fa8bf2f5..e3d6388d 100644 --- a/backend/routes/core/services.py +++ b/backend/routes/core/services.py @@ -34,28 +34,6 @@ async def _dispose_services(): logger.warning(error_msg, exc_info=True) disposal_errors.append(error_msg) - # Dispose watcher - watcher = _services.get("watcher") - if watcher: - try: - watcher.stop() - logger.debug("File watcher stopped successfully") - except Exception as exc: - error_msg = f"Error stopping watcher: {exc}" - logger.warning(error_msg, exc_info=True) - disposal_errors.append(error_msg) - - # Stop filesystem list cache watcher (watchdog) if any. - try: - from backend.adapters.fs.list_cache_watcher import stop_global_fs_list_cache_watcher - - stop_global_fs_list_cache_watcher() - except Exception as exc: - try: - logger.debug("Error stopping fs cache watcher: %s", exc) - except Exception: - pass - # Clear services reference _services = None @@ -74,15 +52,23 @@ async def _build_services(force: bool = False): await _dispose_services() try: - _services = await build_services() - _services_error = None - return _services + services_result = await build_services() except Exception as exc: _services_error = str(exc) logger.error(f"Failed to initialize services: {exc}", exc_info=True) _services = None return None + if not services_result or not getattr(services_result, "ok", False): + _services_error = getattr(services_result, "error", None) or "Initialization failed" + logger.error("Failed to initialize services: %s", _services_error) + _services = None + return None + + _services = services_result.data + _services_error = None + return _services + async def _require_services(): services = await _build_services() diff --git a/backend/routes/handlers/__init__.py b/backend/routes/handlers/__init__.py index cd12e685..038139e2 100644 --- a/backend/routes/handlers/__init__.py +++ b/backend/routes/handlers/__init__.py @@ -12,6 +12,7 @@ from .batch_zip import register_batch_zip_routes from .calendar import register_calendar_routes from .viewer import register_viewer_routes from .db_maintenance import register_db_maintenance_routes +from .releases import register_releases_routes __all__ = [ "register_health_routes", @@ -25,5 +26,6 @@ __all__ = [ "register_calendar_routes", "register_viewer_routes", "register_db_maintenance_routes", + "register_releases_routes", "register_download_routes", ] diff --git a/backend/routes/handlers/collections.py b/backend/routes/handlers/collections.py index 74d82177..147c78f6 100644 --- a/backend/routes/handlers/collections.py +++ b/backend/routes/handlers/collections.py @@ -68,8 +68,8 @@ def _minimal_asset_from_item(item: Dict[str, Any]) -> Dict[str, Any]: "duration": None, "rating": 0, "tags": [], - "has_workflow": 0, - "has_generation_data": 0, + "has_workflow": None, + "has_generation_data": None, "type": asset_type, "root_id": root_id, } @@ -166,9 +166,8 @@ def register_collections_routes(routes: web.RouteTableDef) -> None: a["id"] = db_row.get("id") a["rating"] = int(db_row.get("rating") or 0) a["tags"] = db_row.get("tags") or [] - a["has_workflow"] = int(db_row.get("has_workflow") or 0) - has_gen = int(db_row.get("has_generation_data") or 0) - a["has_generation_data"] = has_gen + a["has_workflow"] = db_row.get("has_workflow") + a["has_generation_data"] = db_row.get("has_generation_data") if db_row.get("root_id"): a["root_id"] = db_row.get("root_id") except Exception as exc: diff --git a/backend/routes/handlers/custom_roots.py b/backend/routes/handlers/custom_roots.py index 9a56ff23..946ae50f 100644 --- a/backend/routes/handlers/custom_roots.py +++ b/backend/routes/handlers/custom_roots.py @@ -2,6 +2,7 @@ Custom roots management endpoints. """ import os +import errno from pathlib import Path from aiohttp import web from backend.custom_roots import ( @@ -11,8 +12,17 @@ from backend.custom_roots import ( resolve_custom_root, ) from backend.shared import Result, get_logger -from backend.adapters.fs.list_cache_watcher import ensure_fs_list_cache_watching -from ..core import _json_response, _csrf_error, _safe_rel_path, _is_within_root, _read_json, _guess_content_type_for_file, _is_allowed_view_media_file +from ..core import ( + _json_response, + _csrf_error, + _safe_rel_path, + _is_within_root, + _read_json, + _guess_content_type_for_file, + _is_allowed_view_media_file, + _require_services, +) +from ..core.security import _check_rate_limit from .filesystem import _kickoff_background_scan # Import tkinter only when needed to avoid startup issues @@ -37,6 +47,10 @@ def register_custom_roots_routes(routes: web.RouteTableDef) -> None: import sys import os + allowed, retry_after = _check_rate_limit(request, "browse_folder", max_requests=3, window_seconds=30) + if not allowed: + return _json_response(Result.Err("RATE_LIMIT", "Too many browse requests", retry_after=retry_after)) + # Check if tkinter is available if tk is None or filedialog is None: logger.error("Tkinter is not available in this environment") @@ -95,17 +109,21 @@ def register_custom_roots_routes(routes: web.RouteTableDef) -> None: label = body.get("label") result = add_custom_root(str(path or ""), label=str(label) if label is not None else None) if result.ok and isinstance(result.data, dict): + root_path = str(result.data.get("path") or "") + root_id = str(result.data.get("id") or "") + # Add to watcher if available + try: + svc, _ = await _require_services() + watcher = svc.get("watcher") if svc else None + if watcher and root_path: + watcher.add_path(root_path, source="custom", root_id=root_id) + except Exception: + pass + # Trigger background scan try: - root_path = str(result.data.get("path") or "") - root_id = str(result.data.get("id") or "") if root_path and root_id: await _kickoff_background_scan(root_path, source="custom", root_id=root_id, recursive=True, incremental=True) - try: - ensure_fs_list_cache_watching(root_path) - except Exception: - pass except Exception as exc: - # Best-effort: never fail the request because background scan scheduling failed. logger.debug("Background scan kickoff skipped: %s", exc) return _json_response(result) @@ -121,7 +139,32 @@ def register_custom_roots_routes(routes: web.RouteTableDef) -> None: body = body_res.data or {} rid = body.get("id") or body.get("root_id") + root_path = None + try: + resolved = resolve_custom_root(str(rid or "")) + if resolved.ok: + root_path = resolved.data + except Exception: + root_path = None + result = remove_custom_root(str(rid or "")) + if result.ok and root_path: + try: + svc, _ = await _require_services() + if svc: + # Remove from watcher + watcher = svc.get("watcher") + if watcher: + watcher.remove_path(str(root_path)) + # Clean up database + db = svc.get("db") + if db: + await db.aexecute( + "DELETE FROM assets WHERE source = 'custom' AND root_id = ?", + (str(rid or ""),), + ) + except Exception: + pass return _json_response(result) @routes.get("/mjr/am/custom-view") @@ -160,8 +203,24 @@ def register_custom_roots_routes(routes: web.RouteTableDef) -> None: if not _is_within_root(candidate, root_dir): return _json_response(Result.Err("INVALID_INPUT", "Path outside root")) - # Fix TOCTOU: Use FileResponse directly and let it handle file existence - # This avoids race condition between exists() check and file serving + def _validate_no_symlink_open(path: Path) -> bool: + try: + flags = os.O_RDONLY + if os.name == "nt" and hasattr(os, "O_BINARY"): + flags |= os.O_BINARY + # Best-effort: disallow following symlinks where supported. + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(str(path), flags) + os.close(fd) + return True + except OSError as exc: + if getattr(exc, "errno", None) in (errno.ELOOP, errno.EACCES, errno.EPERM): + return False + return False + except Exception: + return False + try: resolved_path = candidate.resolve(strict=True) # Prevent symlink/junction escapes: ensure the resolved target is still under the root. @@ -169,6 +228,8 @@ def register_custom_roots_routes(routes: web.RouteTableDef) -> None: return _json_response(Result.Err("FORBIDDEN", "Path escapes root")) if not resolved_path.is_file(): return _json_response(Result.Err("NOT_FOUND", "File not found or not a regular file")) + if not _validate_no_symlink_open(resolved_path): + return _json_response(Result.Err("FORBIDDEN", "Symlinked file not allowed")) # Viewer hardening: only serve image/video media files from custom roots. if not _is_allowed_view_media_file(resolved_path): diff --git a/backend/routes/handlers/db_maintenance.py b/backend/routes/handlers/db_maintenance.py index a2edce04..742973b0 100644 --- a/backend/routes/handlers/db_maintenance.py +++ b/backend/routes/handlers/db_maintenance.py @@ -60,7 +60,7 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None: It is gated behind the same security checks as /mjr/am/index/reset. """ import asyncio - from backend.config import WATCHER_PATHS + from backend.config import OUTPUT_ROOT_PATH from ..core.security import _require_operation_enabled, _require_write_access, _resolve_security_prefs csrf = _csrf_error(request) @@ -94,8 +94,12 @@ def register_db_maintenance_routes(routes: web.RouteTableDef) -> None: # 2. Trigger Rescan started_scans = [] - for path in WATCHER_PATHS: - asyncio.create_task(index_service.scan_directory(str(path), recursive=True, incremental=False)) - started_scans.append(str(path)) + try: + base_path = str(OUTPUT_ROOT_PATH) + except Exception: + base_path = "" + if base_path: + asyncio.create_task(index_service.scan_directory(base_path, recursive=True, incremental=False)) + started_scans.append(base_path) return _json_response(Result.Ok({"reset": True, "scans_triggered": started_scans, "file_ops": (reset_res.meta or {})})) diff --git a/backend/routes/handlers/filesystem.py b/backend/routes/handlers/filesystem.py index 951c16a0..3e20fec3 100644 --- a/backend/routes/handlers/filesystem.py +++ b/backend/routes/handlers/filesystem.py @@ -12,14 +12,13 @@ from pathlib import Path from typing import Any, Mapping, Optional from backend.shared import Result, classify_file, get_logger -from backend.adapters.fs.list_cache_watcher import ensure_fs_list_cache_watching, get_fs_list_cache_token from backend.config import ( FS_LIST_CACHE_MAX, FS_LIST_CACHE_TTL_SECONDS, - FS_LIST_CACHE_WATCHDOG, BG_SCAN_FAILURE_HISTORY_MAX, SCAN_PENDING_MAX, MANUAL_BG_SCAN_GRACE_SECONDS, + BG_SCAN_ON_LIST, ) from shared.scan_throttle import normalize_scan_directory, should_skip_background_scan from ..core import _safe_rel_path, _is_within_root, _require_services @@ -41,6 +40,18 @@ _FS_LIST_CACHE_LOCK = asyncio.Lock() _FS_LIST_CACHE: OrderedDict[str, dict[str, Any]] = OrderedDict() +def _is_truthy_boolish(value: Any) -> bool: + if value is True: + return True + if value is False or value is None: + return False + if value in (1, "1"): + return True + if value in (0, "0"): + return False + return False + + async def _kickoff_background_scan( directory: str, *, @@ -57,6 +68,8 @@ async def _kickoff_background_scan( Used to ensure input/custom folders get indexed without blocking list requests. """ + if not BG_SCAN_ON_LIST: + return normalized_dir = normalize_scan_directory(directory) if should_skip_background_scan(normalized_dir, source, root_id, MANUAL_BG_SCAN_GRACE_SECONDS): return @@ -236,22 +249,11 @@ async def _fs_cache_get_or_build( root_id: Optional[str], ) -> Result[dict[str, Any]]: # Made async to allow async locking if needed, though mostly sync logic - try: - if FS_LIST_CACHE_WATCHDOG: - ensure_fs_list_cache_watching(str(base)) - except Exception: - pass - try: dir_mtime_ns = target_dir_resolved.stat().st_mtime_ns except OSError as exc: return Result.Err("DIR_NOT_FOUND", f"Directory not found: {exc}") - try: - watch_token = get_fs_list_cache_token(str(base)) if FS_LIST_CACHE_WATCHDOG else 0 - except Exception: - watch_token = 0 - cache_key = f"{str(base)}|{str(target_dir_resolved)}|{asset_type}|{str(root_id or '')}" async with _FS_LIST_CACHE_LOCK: @@ -259,7 +261,6 @@ async def _fs_cache_get_or_build( if ( cached and cached.get("dir_mtime_ns") == dir_mtime_ns - and cached.get("watch_token") == watch_token and isinstance(cached.get("entries"), list) ): try: @@ -310,8 +311,8 @@ async def _fs_cache_get_or_build( "duration": None, "rating": 0, "tags": [], - "has_workflow": 0, - "has_generation_data": 0, + "has_workflow": None, + "has_generation_data": None, "type": asset_type, "root_id": root_id, } @@ -320,9 +321,10 @@ async def _fs_cache_get_or_build( return Result.Err("LIST_FAILED", f"Failed to list directory: {exc}") async with _FS_LIST_CACHE_LOCK: + # `watch_token` may be implemented later (fs watch integration); keep None for now. _FS_LIST_CACHE[cache_key] = { "dir_mtime_ns": dir_mtime_ns, - "watch_token": watch_token, + "watch_token": None, "entries": entries, "cached_at": time.time(), } @@ -419,9 +421,8 @@ async def _list_filesystem_assets( asset["id"] = db_row.get("id") asset["rating"] = int(db_row.get("rating") or 0) asset["tags"] = db_row.get("tags") or [] - asset["has_workflow"] = int(db_row.get("has_workflow") or 0) - has_gen = int(db_row.get("has_generation_data") or 0) - asset["has_generation_data"] = has_gen + asset["has_workflow"] = db_row.get("has_workflow") + asset["has_generation_data"] = db_row.get("has_generation_data") # Prefer stored root_id if present (custom) if db_row.get("root_id"): asset["root_id"] = db_row.get("root_id") @@ -440,7 +441,7 @@ async def _list_filesystem_assets( continue if filter_min_rating > 0 and int(item.get("rating", 0)) < filter_min_rating: continue - if filter_workflow_only and not int(item.get("has_workflow", 0)): + if filter_workflow_only and not _is_truthy_boolish(item.get("has_workflow")): continue try: entry_mtime = int(item.get("mtime") or 0) diff --git a/backend/routes/handlers/filesystem_new.py b/backend/routes/handlers/filesystem_new.py deleted file mode 100644 index a3dcb05d..00000000 --- a/backend/routes/handlers/filesystem_new.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -Filesystem operations: background scanning and file listing. -""" - -from __future__ import annotations - -import asyncio -import datetime -import time -from collections import OrderedDict -from pathlib import Path -from typing import Any, Mapping, Optional - -from backend.shared import Result, classify_file, get_logger -from backend.adapters.fs.list_cache_watcher import ensure_fs_list_cache_watching, get_fs_list_cache_token -from backend.config import ( - FS_LIST_CACHE_MAX, - FS_LIST_CACHE_TTL_SECONDS, - FS_LIST_CACHE_WATCHDOG, - BG_SCAN_FAILURE_HISTORY_MAX, - SCAN_PENDING_MAX, - MANUAL_BG_SCAN_GRACE_SECONDS, -) -from shared.scan_throttle import normalize_scan_directory, should_skip_background_scan -from ..core import _safe_rel_path, _is_within_root, _require_services - -logger = get_logger(__name__) - -# Locks for async concurrency safety -_BACKGROUND_SCAN_LOCK = asyncio.Lock() -_BACKGROUND_SCAN_LAST: dict[str, dict[str, Any]] = {} -_BACKGROUND_SCAN_FAILURES: list[dict[str, Any]] = [] -_MAX_FAILURE_HISTORY = int(BG_SCAN_FAILURE_HISTORY_MAX) - -_SCAN_PENDING_LOCK = asyncio.Lock() -_SCAN_PENDING: OrderedDict[str, dict[str, Any]] = OrderedDict() -_SCAN_TASK: Optional[asyncio.Task] = None -_SCAN_PENDING_MAX = int(SCAN_PENDING_MAX) -_WORKER_SHUTDOWN = asyncio.Event() - -_FS_LIST_CACHE_LOCK = asyncio.Lock() -_FS_LIST_CACHE: OrderedDict[str, dict[str, Any]] = OrderedDict() - - -async def _kickoff_background_scan( - directory: str, - *, - source: str = "output", - root_id: Optional[str] = None, - recursive: bool = True, - incremental: bool = True, - fast: bool = True, - background_metadata: bool = True, - min_interval_seconds: float = 10.0, -) -> None: - """ - Fire-and-forget indexing into the DB. - - Used to ensure input/custom folders get indexed without blocking list requests. - """ - normalized_dir = normalize_scan_directory(directory) - if should_skip_background_scan(normalized_dir, source, root_id, MANUAL_BG_SCAN_GRACE_SECONDS): - return - - try: - key = f"{source}|{str(root_id or '')}|{normalized_dir}" - now = time.time() - - entry = _ensure_background_entry(key) - if entry["in_progress"]: - return - if now - entry["last"] < min_interval_seconds: - return - entry["last"] = now - entry["in_progress"] = False - - except Exception: - return - - job = { - "directory": str(normalized_dir), - "source": str(source), - "root_id": str(root_id) if root_id is not None else None, - "recursive": bool(recursive), - "incremental": bool(incremental), - "fast": bool(fast), - "background_metadata": bool(background_metadata), - } - - _ensure_worker() - - async with _SCAN_PENDING_LOCK: - # Move to end if exists (refresh priority) - if key in _SCAN_PENDING: - del _SCAN_PENDING[key] - _SCAN_PENDING[key] = job - - # Enforce max pending size - while len(_SCAN_PENDING) > _SCAN_PENDING_MAX: - _SCAN_PENDING.popitem(last=False) - - -def _ensure_worker() -> None: - global _SCAN_TASK - if _SCAN_TASK and not _SCAN_TASK.done(): - return - - _SCAN_TASK = asyncio.create_task(_worker_loop()) - - -def shutdown_worker() -> None: - _WORKER_SHUTDOWN.set() - - -async def _worker_loop() -> None: - while not _WORKER_SHUTDOWN.is_set(): - task = None - async with _SCAN_PENDING_LOCK: - if _SCAN_PENDING: - _, task = _SCAN_PENDING.popitem(last=False) - else: - task = None - - if not task: - try: - await asyncio.wait_for(_WORKER_SHUTDOWN.wait(), timeout=0.5) - except asyncio.TimeoutError: - pass - continue - - entry: Optional[dict[str, Any]] = None - try: - key = _build_task_key(task) - entry = _ensure_background_entry(key) - entry["in_progress"] = True - entry["last"] = time.time() - - # Import strictly locally or use the core helper to avoid circular imports if possible - # But here we are effectively inside backend.routes.handlers - # We can use the helper from core. - svc, error_result = await _require_services() - if error_result: - _record_scan_failure(str(task.get("directory")), str(task.get("source")), "SERVICE_UNAVAILABLE", str(error_result.error or "")) - await asyncio.sleep(2) # Backoff if services unavailable - continue - - try: - try: - await svc["index"].scan_directory( - str(task.get("directory")), - bool(task.get("recursive")), - bool(task.get("incremental")), - source=str(task.get("source") or "output"), - root_id=task.get("root_id"), - fast=bool(task.get("fast")), - background_metadata=bool(task.get("background_metadata")), - ) - except TypeError: - # Fallback in case of call signature mismatch - await svc["index"].scan_directory( - str(task.get("directory")), - bool(task.get("recursive")), - bool(task.get("incremental")), - source=str(task.get("source") or "output"), - root_id=task.get("root_id"), - ) - except Exception as exc: - _record_scan_failure(str(task.get("directory")), str(task.get("source")), "SCAN_FAILED", str(exc)) - logger.warning("Background scan failed: %s", exc) - - except Exception: - try: - _record_scan_failure(str(task.get("directory")), str(task.get("source")), "EXCEPTION", "Unhandled exception") - except Exception: - pass - finally: - if entry is not None: - entry["in_progress"] = False - entry["last"] = time.time() - - # Yield to event loop - await asyncio.sleep(0) - - -def _record_scan_failure(directory: str, source: str, code: str, error: str) -> None: - # No async lock needed for simple append/pop in main thread - _BACKGROUND_SCAN_FAILURES.insert(0, { - "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "directory": directory, - "source": source, - "code": code, - "error": error - }) - # Trim - while len(_BACKGROUND_SCAN_FAILURES) > _MAX_FAILURE_HISTORY: - _BACKGROUND_SCAN_FAILURES.pop() - - -def _ensure_background_entry(key: str) -> dict[str, Any]: - entry = _BACKGROUND_SCAN_LAST.get(key) - if isinstance(entry, dict): - return entry - if isinstance(entry, (int, float)): - entry = {"last": float(entry), "in_progress": False} - else: - entry = {"last": 0.0, "in_progress": False} - _BACKGROUND_SCAN_LAST[key] = entry - return entry - - -def _build_task_key(task: dict[str, Any]) -> str: - source = str(task.get("source") or "output") - root_id = str(task.get("root_id") or "") - directory = normalize_scan_directory(str(task.get("directory") or "")) - return f"{source}|{root_id}|{directory}" - - -async def _fs_cache_get_or_build( - base: Path, - target_dir_resolved: Path, - asset_type: str, - root_id: Optional[str], -) -> Result[dict[str, Any]]: - # Made async to allow async locking if needed, though mostly sync logic - try: - if FS_LIST_CACHE_WATCHDOG: - ensure_fs_list_cache_watching(str(base)) - except Exception: - pass - - try: - dir_mtime_ns = target_dir_resolved.stat().st_mtime_ns - except OSError as exc: - return Result.Err("DIR_NOT_FOUND", f"Directory not found: {exc}") - - try: - watch_token = get_fs_list_cache_token(str(base)) if FS_LIST_CACHE_WATCHDOG else 0 - except Exception: - watch_token = 0 - - cache_key = f"{str(base)}|{str(target_dir_resolved)}|{asset_type}|{str(root_id or '')}" - - async with _FS_LIST_CACHE_LOCK: - cached = _FS_LIST_CACHE.get(cache_key) - if ( - cached - and cached.get("dir_mtime_ns") == dir_mtime_ns - and cached.get("watch_token") == watch_token - and isinstance(cached.get("entries"), list) - ): - try: - now = time.time() - cached_at = float(cached.get("cached_at") or 0.0) - if cached_at and (now - cached_at) <= float(FS_LIST_CACHE_TTL_SECONDS): - _FS_LIST_CACHE.move_to_end(cache_key) - return Result.Ok({"entries": cached["entries"], "dir_mtime_ns": dir_mtime_ns}) - except Exception: - pass - _FS_LIST_CACHE.move_to_end(cache_key) - - entries: list[dict[str, Any]] = [] - try: - # Note: os.scandir/iterdir is synchronous I/O. - # For large directories, this will block the loop. - # Ideally this should run in a thread executor, but for now we keep it simple. - for entry in target_dir_resolved.iterdir(): - if not entry.is_file(): - continue - filename = entry.name - kind = classify_file(filename) - if kind == "unknown": - continue - try: - stat = entry.stat() - except OSError: - continue - - try: - rel_to_root = entry.parent.relative_to(base) - sub = "" if str(rel_to_root) == "." else str(rel_to_root).replace("\\", "/") - except ValueError: - sub = "" - - entries.append( - { - "id": None, - "filename": filename, - "subfolder": sub, - "filepath": str(entry), - "kind": kind, - "ext": entry.suffix.lower(), - "size": stat.st_size, - "mtime": int(stat.st_mtime), - "width": None, - "height": None, - "duration": None, - "rating": 0, - "tags": [], - "has_workflow": 0, - "has_generation_data": 0, - "type": asset_type, - "root_id": root_id, - } - ) - except OSError as exc: - return Result.Err("LIST_FAILED", f"Failed to list directory: {exc}") - - async with _FS_LIST_CACHE_LOCK: - _FS_LIST_CACHE[cache_key] = { - "dir_mtime_ns": dir_mtime_ns, - "watch_token": watch_token, - "entries": entries, - "cached_at": time.time(), - } - _FS_LIST_CACHE.move_to_end(cache_key) - while len(_FS_LIST_CACHE) > int(FS_LIST_CACHE_MAX): - _FS_LIST_CACHE.popitem(last=False) - - return Result.Ok({"entries": entries, "dir_mtime_ns": dir_mtime_ns}) - - -async def _list_filesystem_assets( - root_dir: Path, - subfolder: str, - query: str, - limit: int, - offset: int, - asset_type: str, - root_id: Optional[str] = None, - filters: Optional[Mapping[str, Any]] = None, - index_service: Optional[Any] = None, -) -> Result[dict[str, Any]]: - try: - base = root_dir.resolve() - except OSError as exc: - return Result.Err("INVALID_INPUT", f"Invalid root directory: {exc}") - - rel = _safe_rel_path(subfolder or "") - if rel is None: - return Result.Err("INVALID_INPUT", "Invalid subfolder") - - target_dir = (base / rel) - try: - target_dir_resolved = target_dir.resolve(strict=True) - except OSError: - return Result.Err("DIR_NOT_FOUND", f"Directory not found: {target_dir}") - - if not _is_within_root(target_dir_resolved, base): - return Result.Err("INVALID_INPUT", "Subfolder outside root") - - q = (query or "*").strip() - q_lower = q.lower() - browse_all = q == "*" or q == "" - - filter_kind = str(filters.get("kind") or "").strip().lower() if filters is not None else "" - filter_min_rating = int(filters.get("min_rating") or 0) if filters is not None else 0 - filter_workflow_only = bool(filters.get("has_workflow")) if filters is not None else False - - mtime_start_raw = filters.get("mtime_start") if filters is not None else None - mtime_end_raw = filters.get("mtime_end") if filters is not None else None - filter_mtime_start = int(mtime_start_raw) if mtime_start_raw is not None else None - filter_mtime_end = int(mtime_end_raw) if mtime_end_raw is not None else None - - # Changed to await the async cache builder - cache_result = await _fs_cache_get_or_build(base, target_dir_resolved, asset_type, root_id) - if not cache_result.ok: - return cache_result - all_entries = cache_result.data.get("entries") if isinstance(cache_result.data, dict) else None - if not isinstance(all_entries, list): - return Result.Err("LIST_FAILED", "Failed to list directory") - - entries = [] - for item in all_entries: - if not isinstance(item, dict): - continue - filename = str(item.get("filename") or "") - kind = str(item.get("kind") or "") - if not filename or kind == "unknown": - continue - if filter_kind and kind != filter_kind: - continue - if not browse_all and q_lower not in filename.lower(): - continue - entries.append(item) - - entries.sort(key=lambda x: x.get("mtime") or 0, reverse=True) - - # Enrich with DB fields (workflow/generation/tags/rating) when available. - try: - lookup = getattr(index_service, "lookup_assets_by_filepaths", None) - if callable(lookup) and entries: - filepaths = [str(a.get("filepath") or "") for a in entries if isinstance(a, dict)] - filepaths = [p for p in filepaths if p] - if filepaths: - enrich_result = await lookup(filepaths) - if enrich_result and getattr(enrich_result, "ok", False) and isinstance(enrich_result.data, dict): - mapping = enrich_result.data - for asset in entries: - if not isinstance(asset, dict): - continue - fp = str(asset.get("filepath") or "") - db_row = mapping.get(fp) - if not isinstance(db_row, dict): - continue - asset["id"] = db_row.get("id") - asset["rating"] = int(db_row.get("rating") or 0) - asset["tags"] = db_row.get("tags") or [] - asset["has_workflow"] = int(db_row.get("has_workflow") or 0) - has_gen = int(db_row.get("has_generation_data") or 0) - asset["has_generation_data"] = has_gen - # Prefer stored root_id if present (custom) - if db_row.get("root_id"): - asset["root_id"] = db_row.get("root_id") - except Exception as exc: - logger.debug("Filesystem DB enrichment skipped: %s", exc) - - # Apply filters after enrichment. - total = 0 - paged: list[dict[str, Any]] = [] - start = max(0, int(offset or 0)) - limit_int = int(limit or 0) - end = start + limit_int if limit_int > 0 else None - - for item in entries: - if filter_kind and item.get("kind") != filter_kind: - continue - if filter_min_rating > 0 and int(item.get("rating", 0)) < filter_min_rating: - continue - if filter_workflow_only and not int(item.get("has_workflow", 0)): - continue - try: - entry_mtime = int(item.get("mtime") or 0) - except Exception: - entry_mtime = 0 - if filter_mtime_start is not None and entry_mtime < filter_mtime_start: - continue - if filter_mtime_end is not None and entry_mtime >= filter_mtime_end: - continue - if not browse_all and q_lower not in str(item.get("filename", "")).lower(): - continue - - # This entry passed all filters; count it for total. - if total >= start: - if end is None or len(paged) < (end - start): - paged.append(item) - total += 1 - - return Result.Ok({"assets": paged, "total": total, "limit": limit, "offset": offset, "query": q}) diff --git a/backend/routes/handlers/health.py b/backend/routes/handlers/health.py index 50628cf9..7e55eec1 100644 --- a/backend/routes/handlers/health.py +++ b/backend/routes/handlers/health.py @@ -26,6 +26,7 @@ from ..core import _json_response, _require_services, _csrf_error, _require_writ SECURITY_PREF_KEYS = { "safe_mode", "allow_write", + "allow_remote_write", "allow_delete", "allow_rename", "allow_open_in_folder", @@ -89,6 +90,17 @@ def register_health_routes(routes: web.RouteTableDef) -> None: result.data["scope"] = scope if scope == "custom": result.data["custom_root_id"] = custom_root_id + try: + watcher = svc.get("watcher") if isinstance(svc, dict) else None + watcher_scope = svc.get("watcher_scope") if isinstance(svc, dict) else None + result.data["watcher"] = { + "enabled": bool(watcher is not None and getattr(watcher, "is_running", False)), + "directories": watcher.watched_directories if watcher else [], + "scope": (watcher_scope or {}).get("scope") if isinstance(watcher_scope, dict) else None, + "custom_root_id": (watcher_scope or {}).get("custom_root_id") if isinstance(watcher_scope, dict) else None, + } + except Exception: + result.data["watcher"] = {"enabled": False, "directories": [], "scope": None, "custom_root_id": None} return _json_response(result) @routes.get("/mjr/am/config") diff --git a/backend/routes/handlers/releases.py b/backend/routes/handlers/releases.py new file mode 100644 index 00000000..11302338 --- /dev/null +++ b/backend/routes/handlers/releases.py @@ -0,0 +1,74 @@ +""" +Expose repository refs (tags/branches) and a ZIP template for programmatic installs. + +Endpoint: GET /mjr/am/releases +Query params: + - owner (default: MajoorWaldi) + - repo (default: ComfyUI-Majoor-AssetsManager) + - per_page (default: 100) + +Returns a `Result` with `data` containing `tags`, `branches`, and `zip_url_template`. +If GitHub is unreachable, returns a `Result.Err` with code `DEGRADED`. +""" +from __future__ import annotations + +import os +import asyncio +from typing import Any + +from aiohttp import web, ClientSession + +from backend.shared import Result, get_logger +from ..core import _json_response + +logger = get_logger(__name__) + + +def register_releases_routes(routes: web.RouteTableDef) -> None: + @routes.get("/mjr/am/releases") + async def get_releases(request: web.Request) -> web.Response: + owner = (request.query.get("owner") or "MajoorWaldi").strip() + repo = (request.query.get("repo") or "ComfyUI-Majoor-AssetsManager").strip() + try: + per_page = int(request.query.get("per_page") or "100") + except Exception: + per_page = 100 + + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("MAJOOR_GITHUB_TOKEN") + headers = {"Accept": "application/vnd.github.v3+json"} + if token: + headers["Authorization"] = f"token {token}" + + tags_url = f"https://api.github.com/repos/{owner}/{repo}/tags?per_page={per_page}" + branches_url = f"https://api.github.com/repos/{owner}/{repo}/branches?per_page={per_page}" + + try: + async with ClientSession() as session: + async def fetch(url: str) -> Any: + async with session.get(url, headers=headers, timeout=30) as resp: + if resp.status != 200: + text = await resp.text() + raise RuntimeError(f"GitHub API returned {resp.status}: {text}") + return await resp.json() + + tags_json, branches_json = await asyncio.gather(fetch(tags_url), fetch(branches_url)) + + tags = [t.get("name") for t in (tags_json or []) if isinstance(t, dict) and t.get("name")] + branches = [b.get("name") for b in (branches_json or []) if isinstance(b, dict) and b.get("name")] + + zip_url_template = f"https://github.com/{owner}/{repo}/archive/refs/{{ref}}.zip" + + data = { + "owner": owner, + "repo": repo, + "tags": tags, + "branches": branches, + "zip_url_template": zip_url_template, + } + return _json_response(Result.Ok(data)) + except asyncio.TimeoutError: + logger.debug("Timeout fetching GitHub refs for %s/%s", owner, repo) + return _json_response(Result.Err("DEGRADED", "Timeout while contacting GitHub")) + except Exception as exc: + logger.debug("Failed to fetch GitHub refs for %s/%s: %s", owner, repo, exc) + return _json_response(Result.Err("DEGRADED", "Failed to fetch repository refs", meta={"details": str(exc)})) diff --git a/backend/routes/handlers/scan.py b/backend/routes/handlers/scan.py index bdee3eeb..e4b67a79 100644 --- a/backend/routes/handlers/scan.py +++ b/backend/routes/handlers/scan.py @@ -31,6 +31,13 @@ from backend.config import ( from backend.custom_roots import resolve_custom_root from backend.shared import Result, ErrorCode, get_logger from backend.utils import parse_bool +from backend.features.index.metadata_helpers import MetadataHelpers +from backend.features.index.watcher_scope import ( + build_watch_paths, + normalize_scope, + WATCHER_SCOPE_KEY, + WATCHER_CUSTOM_ROOT_ID_KEY, +) from ..core import ( _json_response, _require_services, @@ -144,7 +151,8 @@ async def _write_multipart_file_atomic(dest_dir: Path, filename: str, field) -> total += len(chunk) if total > _MAX_UPLOAD_SIZE: raise ValueError(f"File exceeds maximum size ({_MAX_UPLOAD_SIZE} bytes)") - f.write(chunk) + # Avoid blocking the event loop on large writes. + await asyncio.to_thread(f.write, chunk) tmp_obj = Path(str(tmp_path)) final = dest_dir / safe_name @@ -362,10 +370,14 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: out_res = results[0] in_res = results[1] - - # Handle exceptions if any - if isinstance(out_res, Exception): raise out_res - if isinstance(in_res, Exception): raise in_res + + # Handle exceptions if any (never raise to UI). + if isinstance(out_res, Exception): + logger.error("Output scan failed: %s", out_res) + return _json_response(Result.Err("SCAN_FAILED", safe_error_message(out_res, "Output scan failed"))) + if isinstance(in_res, Exception): + logger.error("Input scan failed: %s", in_res) + return _json_response(Result.Err("SCAN_FAILED", safe_error_message(in_res, "Input scan failed"))) except asyncio.TimeoutError: return _json_response(Result.Err("TIMEOUT", "Scan timed out")) @@ -507,12 +519,14 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: body = body_res.data or {} files = body.get("files") + origin = str(body.get("origin") or body.get("source") or "").strip().lower() if not isinstance(files, list) or not files: result = Result.Err("INVALID_INPUT", "Missing or invalid 'files' list") return _json_response(result) incremental = body.get("incremental", True) grouped_paths: Dict[Tuple[str, str, str], list] = {} + recent_generated_paths: list[str] = [] pending_ops: list[dict[str, Any]] = [] @@ -521,10 +535,92 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: continue filename = item.get("filename") or item.get("name") if not filename: - continue + filename = None subfolder = item.get("subfolder") or "" file_type = (item.get("type") or "output").lower() root_id = item.get("root_id") or item.get("custom_root_id") + raw_path = item.get("path") or item.get("filepath") or item.get("fullpath") or item.get("full_path") + + # If an absolute path is provided, prefer it (more robust across ComfyUI variants). + if raw_path: + normalized = None + try: + p = Path(str(raw_path)).expanduser() + if p.is_absolute() or getattr(p, "drive", ""): + normalized = p.resolve(strict=True) + except Exception: + normalized = None + + if normalized is not None: + base_root = None + base_type = None + base_root_id = root_id + + try: + out_root = Path(OUTPUT_ROOT).resolve(strict=True) + except Exception: + out_root = None + try: + in_root = Path(folder_paths.get_input_directory()).resolve(strict=True) + except Exception: + in_root = None + + if out_root and _is_within_root(normalized, out_root): + base_root = str(out_root) + base_type = "output" + base_root_id = None + elif in_root and _is_within_root(normalized, in_root): + base_root = str(in_root) + base_type = "input" + base_root_id = None + elif file_type == "custom": + root_result = resolve_custom_root(str(root_id or "")) + if root_result.ok: + try: + custom_root = Path(str(root_result.data)).resolve(strict=True) + except Exception: + custom_root = None + if custom_root and _is_within_root(normalized, custom_root): + base_root = str(custom_root) + base_type = "custom" + else: + try: + if _is_path_allowed_custom(normalized): + from backend.custom_roots import list_custom_roots + + roots_res = list_custom_roots() + if roots_res.ok and isinstance(roots_res.data, list): + for r in roots_res.data: + try: + rp = Path(str(r.get("path") or "")).resolve(strict=True) + if _is_within_root(normalized, rp): + base_root = str(rp) + base_type = "custom" + base_root_id = r.get("id") or root_id + break + except Exception: + continue + except Exception: + pass + + if base_root and base_type: + # Enforce allowlist checks + if base_type == "custom": + if not _is_path_allowed_custom(normalized): + continue + else: + if not _is_path_allowed(normalized): + continue + grouped_paths.setdefault((base_root, base_type, str(base_root_id or "")), []).append(normalized) + if origin in ("generation", "executed", "comfy", "comfyui"): + try: + recent_generated_paths.append(str(normalized)) + except Exception: + pass + continue + + if not filename: + continue if file_type == "input": base_root = folder_paths.get_input_directory() @@ -559,11 +655,23 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: continue grouped_paths.setdefault((base_dir, file_type, str(root_id or "")), []).append(normalized) + if origin in ("generation", "executed", "comfy", "comfyui"): + try: + recent_generated_paths.append(str(normalized)) + except Exception: + pass if not grouped_paths: result = Result.Err("INVALID_INPUT", "No valid files to index") return _json_response(result) + if recent_generated_paths: + try: + from backend.features.index.watcher import mark_recent_generated + mark_recent_generated(recent_generated_paths) + except Exception: + pass + total_stats = { "scanned": 0, "added": 0, @@ -629,7 +737,9 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: try: db_adapter = svc['db'] - async with db_adapter.atransaction(): + async with db_adapter.atransaction() as tx: + if not tx.ok: + raise RuntimeError(tx.error or "Failed to begin transaction") # A. Create Temp Table for Batch Processing await db_adapter.aexecute("CREATE TEMPORARY TABLE IF NOT EXISTS temp_gen_updates (filename TEXT, subfolder TEXT, source TEXT, gen_time INTEGER)") await db_adapter.aexecute("DELETE FROM temp_gen_updates") @@ -669,17 +779,54 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: upsert_params.append((asset_id, new_json)) # D. Bulk Upsert (INSERT or UPDATE) - # Use ON CONFLICT to handle both new and existing rows + # Prefer per-asset locked writes using MetadataHelpers to avoid races + # with other concurrent metadata writers (background enrichers, manual edits). if upsert_params: - await db_adapter.aexecutemany(""" - INSERT INTO asset_metadata (asset_id, metadata_raw) - VALUES (?, ?) - ON CONFLICT(asset_id) DO UPDATE SET - metadata_raw = excluded.metadata_raw - """, upsert_params) + try: + from backend.features.index.metadata_helpers import MetadataHelpers + for asset_id, new_json in upsert_params: + try: + async with db_adapter.lock_for_asset(int(asset_id)): + # Load current metadata_raw for this asset (if any) + cur = await db_adapter.aquery("SELECT metadata_raw FROM asset_metadata WHERE asset_id = ? LIMIT 1", (int(asset_id),)) + cur_meta = {} + if cur.ok and cur.data and cur.data[0].get("metadata_raw"): + try: + cur_meta = json.loads(cur.data[0].get("metadata_raw") or "{}") + except Exception: + cur_meta = {} + + # Parse incoming partial metadata (e.g., {'generation_time_ms': 123}) + incoming_meta = {} + try: + incoming_meta = json.loads(new_json) if new_json else {} + except Exception: + incoming_meta = {} + + # Merge conservatively: preserve existing keys and update with incoming values + merged = dict(cur_meta) + merged.update(incoming_meta) + + # Use MetadataHelpers to write merged metadata safely (it handles quality/tags/flags) + await MetadataHelpers.write_asset_metadata_row(db_adapter, int(asset_id), Result.Ok(merged)) + except Exception: + logger.debug("Failed to upsert generation time for asset %s", asset_id) + except Exception: + # Fallback: if per-asset path fails for any reason, try the original bulk upsert SQL + try: + await db_adapter.aexecutemany(""" + INSERT INTO asset_metadata (asset_id, metadata_raw) + VALUES (?, ?) + ON CONFLICT(asset_id) DO UPDATE SET + metadata_raw = excluded.metadata_raw + """, upsert_params) + except Exception: + logger.debug("Fallback SQL bulk upsert failed") # Cleanup await db_adapter.aexecute("DROP TABLE IF EXISTS temp_gen_updates") + if not tx.ok: + raise RuntimeError(tx.error or "Commit failed") except Exception as ex: logger.debug("Failed during batch metadata enhancement: %s", ex) @@ -964,7 +1111,9 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: async def _do_clears() -> Result[dict]: # Make clears atomic and resistant to concurrent writers. - async with db.atransaction(mode="immediate"): + async with db.atransaction(mode="immediate") as tx: + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") if clear_scan_journal: res = await _clear_table("scan_journal", cache_prefixes) if not res.ok: @@ -988,6 +1137,8 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: if not res.ok: return res cleared["assets"] = int(res.data or 0) + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Commit failed") return Result.Ok(cleared) try: @@ -1007,7 +1158,7 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: if scope == "all" and (clear_scan_journal or clear_metadata_cache): # 1. Vacuum DB to reclaim space and rebuild file try: - await db.aexecute("VACUUM") + await asyncio.to_thread(db.execute, "VACUUM") logger.info("Database VACUUM completed during index reset") except Exception as exc: logger.warning(f"Failed to VACUUM database: {exc}") @@ -1015,7 +1166,7 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: # 2. Cleanup physical stray files in index directory # Verify we are cleaning the right folder and safeguard critical files if reindex and INDEX_DIR_PATH.exists(): - try: + def _cleanup_index_dir(): for item in INDEX_DIR_PATH.iterdir(): if item.name.startswith("assets.sqlite"): continue @@ -1033,6 +1184,9 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: logger.info(f"Deleted stray index item: {item.name}") except Exception as e: logger.warning(f"Failed to delete {item.name}: {e}") + + try: + await asyncio.to_thread(_cleanup_index_dir) except Exception as exc: logger.warning(f"Index directory cleanup failed: {exc}") @@ -1084,8 +1238,12 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: rebuild_status = None if rebuild_fts_flag: try: - async with db.atransaction(mode="immediate"): + async with db.atransaction(mode="immediate") as tx: + if not tx.ok: + return _json_response(Result.Err("DB_ERROR", tx.error or "Failed to begin transaction")) rebuild_result = await rebuild_fts(db) + if not tx.ok: + return _json_response(Result.Err("DB_ERROR", tx.error or "Commit failed")) except Exception as exc: logger.exception("FTS rebuild failed during index reset") return _json_response(Result.Err("DB_ERROR", f"FTS rebuild failed: {exc}")) @@ -1372,9 +1530,8 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: purpose = request.query.get("purpose", "").lower().strip() skip_index = (purpose == "node_drop") - # Get input directory + # Get input directory (use module-level `folder_paths` so tests can monkeypatch) try: - import folder_paths input_dir = Path(folder_paths.get_input_directory()).resolve() except Exception: input_dir = Path(__file__).parent.parent.parent.parent / "input" @@ -1412,4 +1569,179 @@ def register_scan_routes(routes: web.RouteTableDef) -> None: logger.error(f"Upload failed: {e}") return _json_response(Result.Err("UPLOAD_FAILED", "Upload failed")) + @routes.get("/mjr/am/watcher/status") + async def watcher_status(request): + """Get watcher status.""" + svc, error_result = await _require_services() + if error_result: + return _json_response(error_result) + + watcher = svc.get("watcher") + return _json_response(Result.Ok({ + "enabled": watcher is not None and watcher.is_running, + "directories": watcher.watched_directories if watcher else [], + })) + + @routes.post("/mjr/am/watcher/toggle") + async def watcher_toggle(request): + """Start or stop the file watcher.""" + svc, error_result = await _require_services() + if error_result: + return _json_response(error_result) + + csrf = _csrf_error(request) + if csrf: + return _json_response(Result.Err("CSRF", csrf)) + + body_res = await _read_json(request) + if not body_res.ok: + return _json_response(body_res) + body = body_res.data or {} + + enabled = body.get("enabled", True) + watcher = svc.get("watcher") + + if enabled: + # Start watcher + if watcher and watcher.is_running: + return _json_response(Result.Ok({"enabled": True, "directories": watcher.watched_directories})) + + from backend.features.index.watcher import OutputWatcher + + index_service = svc.get("index") + if not index_service: + return _json_response(Result.Err("SERVICE_UNAVAILABLE", "Index service not available")) + + async def index_callback(filepaths, base_dir, source=None, root_id=None): + if not filepaths: + return + try: + # Give executed-event a moment to mark generated files. + await asyncio.sleep(0.2) + except Exception: + pass + try: + from backend.features.index.watcher import is_recent_generated + filepaths = [f for f in (filepaths or []) if f and not is_recent_generated(f)] + except Exception: + pass + if not filepaths: + return + paths = [Path(f) for f in filepaths if f] + if paths: + await index_service.index_paths( + paths=paths, + base_dir=base_dir, + incremental=True, + source=source or "watcher", + root_id=root_id, + ) + + new_watcher = OutputWatcher(index_callback) + scope_cfg = svc.get("watcher_scope") if isinstance(svc, dict) else None + desired_scope = normalize_scope((scope_cfg or {}).get("scope")) + desired_root_id = (scope_cfg or {}).get("custom_root_id") or (scope_cfg or {}).get("root_id") + watch_paths = build_watch_paths(desired_scope, desired_root_id) + + if watch_paths: + loop = asyncio.get_event_loop() + await new_watcher.start(watch_paths, loop) + svc["watcher"] = new_watcher + svc["watcher_scope"] = {"scope": desired_scope, "custom_root_id": desired_root_id or ""} + return _json_response(Result.Ok({"enabled": True, "directories": new_watcher.watched_directories})) + else: + return _json_response(Result.Err("NO_DIRECTORIES", "No directories to watch")) + else: + # Stop watcher + if watcher: + await watcher.stop() + svc["watcher"] = None + return _json_response(Result.Ok({"enabled": False, "directories": []})) + + @routes.post("/mjr/am/watcher/scope") + async def watcher_scope(request): + """Configure watcher to follow a single scope (global mode).""" + svc, error_result = await _require_services() + if error_result: + return _json_response(error_result) + + csrf = _csrf_error(request) + if csrf: + return _json_response(Result.Err("CSRF", csrf)) + + body_res = await _read_json(request) + if not body_res.ok: + return _json_response(body_res) + body = body_res.data or {} + + scope = normalize_scope(body.get("scope")) + custom_root_id = body.get("custom_root_id") or body.get("root_id") or body.get("customRootId") + custom_root_id = str(custom_root_id or "").strip() + + # Persist desired scope in memory even if watcher is disabled. + svc["watcher_scope"] = {"scope": scope, "custom_root_id": custom_root_id} + try: + db = svc.get("db") + if db: + await MetadataHelpers.set_metadata_value(db, WATCHER_SCOPE_KEY, str(scope)) + await MetadataHelpers.set_metadata_value(db, WATCHER_CUSTOM_ROOT_ID_KEY, str(custom_root_id or "")) + except Exception: + pass + + # If watcher is not enabled, just acknowledge. + watcher = svc.get("watcher") + if not watcher or not watcher.is_running: + return _json_response(Result.Ok({"enabled": False, "directories": [], "scope": scope})) + + from backend.features.index.watcher import OutputWatcher + + index_service = svc.get("index") + if not index_service: + return _json_response(Result.Err("SERVICE_UNAVAILABLE", "Index service not available")) + + watch_paths = build_watch_paths(scope, custom_root_id) + if not watch_paths: + try: + if watcher and watcher.is_running: + await watcher.stop() + svc["watcher"] = None + except Exception: + pass + return _json_response(Result.Ok({"enabled": False, "directories": [], "scope": scope})) + + try: + await watcher.stop() + except Exception: + pass + + async def index_callback(filepaths, base_dir, source=None, root_id=None): + if not filepaths: + return + try: + # Give executed-event a moment to mark generated files. + await asyncio.sleep(0.2) + except Exception: + pass + try: + from backend.features.index.watcher import is_recent_generated + filepaths = [f for f in (filepaths or []) if f and not is_recent_generated(f)] + except Exception: + pass + if not filepaths: + return + paths = [Path(f) for f in filepaths if f] + if paths: + await index_service.index_paths( + paths=paths, + base_dir=base_dir, + incremental=True, + source=source or "watcher", + root_id=root_id, + ) + + new_watcher = OutputWatcher(index_callback) + loop = asyncio.get_event_loop() + await new_watcher.start(watch_paths, loop) + svc["watcher"] = new_watcher + return _json_response(Result.Ok({"enabled": True, "directories": new_watcher.watched_directories, "scope": scope})) diff --git a/backend/routes/registry.py b/backend/routes/registry.py index 0ea18d5a..7ee5904d 100644 --- a/backend/routes/registry.py +++ b/backend/routes/registry.py @@ -59,6 +59,7 @@ from .handlers import ( register_calendar_routes, register_viewer_routes, register_db_maintenance_routes, + register_releases_routes, register_download_routes, ) @@ -156,6 +157,12 @@ def register_all_routes() -> web.RouteTableDef: register_calendar_routes(routes) register_viewer_routes(routes) register_db_maintenance_routes(routes) + # Register releases route (exposes tags/branches + zip template) + try: + register_releases_routes(routes) + logger.info(" GET /mjr/am/releases (Added)") + except Exception as e: + logger.error(f"Failed to register releases routes: {e}") # FIX: Enregistrement de la route de téléchargement try: @@ -186,6 +193,7 @@ def register_all_routes() -> web.RouteTableDef: logger.info(" GET /mjr/am/viewer/info?asset_id=") logger.info(" POST /mjr/am/db/optimize") logger.info(" GET /mjr/am/download") + logger.info(" GET /mjr/am/releases") logger.info("=" * 60) return routes diff --git a/backend/settings.py b/backend/settings.py index a05bfd7f..930938cf 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -4,6 +4,7 @@ Application settings persisted in the local metadata store. from __future__ import annotations import threading +import os import asyncio import time from typing import Any, Mapping, Optional @@ -20,6 +21,7 @@ _VALID_PROBE_MODES = {"auto", "exiftool", "ffprobe", "both"} _SECURITY_PREFS_INFO: Mapping[str, dict[str, bool | str]] = { "safe_mode": {"env": "MAJOOR_SAFE_MODE", "default": False}, "allow_write": {"env": "MAJOOR_ALLOW_WRITE", "default": False}, + "allow_remote_write": {"env": "MAJOOR_ALLOW_REMOTE_WRITE", "default": True}, "allow_delete": {"env": "MAJOOR_ALLOW_DELETE", "default": True}, "allow_rename": {"env": "MAJOOR_ALLOW_RENAME", "default": True}, "allow_open_in_folder": {"env": "MAJOOR_ALLOW_OPEN_IN_FOLDER", "default": True}, @@ -92,6 +94,13 @@ class AppSettings: res = await self._write_setting(key, "1" if value else "0") if not res.ok: return Result.Err("DB_ERROR", res.error or f"Failed to persist {key}") + try: + info = _SECURITY_PREFS_INFO.get(key) or {} + env_var = str(info.get("env") or "").strip() + if env_var: + os.environ[env_var] = "1" if value else "0" + except Exception: + pass bump = await self._bump_settings_version_locked() if not bump.ok: try: diff --git a/comfyui_extension.json b/comfyui_extension.json index 1675faac..12918e78 100644 --- a/comfyui_extension.json +++ b/comfyui_extension.json @@ -17,6 +17,13 @@ "search" ], "install_type": "git-clone", + "releases": { + "provider": "github", + "owner": "MajoorWaldi", + "repo": "ComfyUI-Majoor-AssetsManager", + "zip_url_template": "https://github.com/MajoorWaldi/ComfyUI-Majoor-AssetsManager/archive/refs/{ref}.zip", + "notes": "Use {ref}=tags/vX.Y.Z for a release tag or {ref}=heads/main for the nightly branch (heads/)." + }, "js_path": "js", "nodename_pattern": "^Majoor" } diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 51fd3079..3fd96f0a 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -171,11 +171,6 @@ export MAJOOR_FFPROBE_PATH="/path/to/ffprobe" # Set media probe backend (auto, exiftool, ffprobe, both) export MAJOOR_MEDIA_PROBE_BACKEND="auto" -# Enable file watching for automatic reindexing -export MAJOOR_ENABLE_FILE_WATCHER=true - -# Set watcher interval (in seconds) -export MAJOOR_WATCHER_INTERVAL=15.0 ``` ### Windows Batch File Example @@ -184,8 +179,6 @@ Create a batch file to set environment variables and start ComfyUI: ```batch @echo off set MAJOOR_MEDIA_PROBE_BACKEND=auto -set MAJOOR_ENABLE_FILE_WATCHER=true -set MAJOOR_WATCHER_INTERVAL=15.0 REM Start ComfyUI with the environment variables cd /d "C:\path\to\ComfyUI" @@ -272,4 +265,4 @@ Simply click "Uninstall" in the ComfyUI Manager interface. --- *Installation Guide Version: 1.0* -*Last Updated: January 2026* \ No newline at end of file +*Last Updated: January 2026* diff --git a/docs/SETTINGS_CONFIGURATION.md b/docs/SETTINGS_CONFIGURATION.md index 2c804e4e..db3f7099 100644 --- a/docs/SETTINGS_CONFIGURATION.md +++ b/docs/SETTINGS_CONFIGURATION.md @@ -94,15 +94,6 @@ Settings are primarily adjusted through the user interface: ## File System Settings -### File Watching -- **Enable File Watcher**: Automatically reindex when files change -- **Watcher Interval**: How often to check for file changes -- **Join Timeout**: Time to wait before processing multiple changes together -- **Watched Paths**: Which directories to monitor for changes -- **Default**: Usually Disabled to save resources -- **Impact**: Keeps index current but uses system resources -- **Recommendation**: Enable for actively used output directories - ### Custom Roots - **Adding Custom Directories**: Add additional directories to browse - **Path Validation**: Ensures paths are valid and accessible @@ -161,31 +152,6 @@ Settings are primarily adjusted through the user interface: - Impact: Determines which tools are used for metadata extraction - Example: `MAJOOR_MEDIA_PROBE_BACKEND=both` -#### File Watching -- **MAJOOR_ENABLE_FILE_WATCHER**: Enable automatic reindexing - - Options: `true`, `false`, `1`, `0` - - Default: `false` - - Impact: Automatically updates index when files change - - Example: `MAJOOR_ENABLE_FILE_WATCHER=true` - -- **MAJOOR_WATCHER_INTERVAL**: File watching interval in seconds - - Default: 15.0 - - Range: 1.0 to 300.0 - - Impact: How frequently the system checks for changes - - Example: `MAJOOR_WATCHER_INTERVAL=10.0` - -- **MAJOOR_WATCHER_JOIN_TIMEOUT**: Coalescing timeout for file events - - Default: 5.0 seconds - - Range: 0.1 to 60.0 - - Impact: Time to wait before processing grouped file changes - - Example: `MAJOOR_WATCHER_JOIN_TIMEOUT=3.0` - -- **MAJOOR_WATCHER_PATHS**: Directories to watch (colon-separated on Unix, semicolon on Windows) - - Default: Output directory - - Format: Path separators depend on OS - - Impact: Which directories are monitored for changes - - Example: `MAJOOR_WATCHER_PATHS=/path/to/output:/path/to/inputs` - #### Database Tuning - **MAJOOR_DB_TIMEOUT**: Database operation timeout in seconds - Default: 30.0 @@ -253,8 +219,6 @@ Create a batch file to set environment variables: ```batch @echo off set MAJOOR_MEDIA_PROBE_BACKEND=auto -set MAJOOR_ENABLE_FILE_WATCHER=true -set MAJOOR_WATCHER_INTERVAL=15.0 set MAJOOR_DB_TIMEOUT=60.0 REM Start ComfyUI with the environment variables @@ -269,8 +233,6 @@ Create a shell script to set environment variables: ```bash #!/bin/bash export MAJOOR_MEDIA_PROBE_BACKEND=auto -export MAJOOR_ENABLE_FILE_WATCHER=true -export MAJOOR_WATCHER_INTERVAL=15.0 export MAJOOR_DB_TIMEOUT=60.0 # Start ComfyUI with the environment variables @@ -282,8 +244,6 @@ Or add to your shell profile (`.bashrc`, `.zshrc`, etc.): ```bash export MAJOOR_MEDIA_PROBE_BACKEND=auto -export MAJOOR_ENABLE_FILE_WATCHER=true -export MAJOOR_WATCHER_INTERVAL=15.0 export MAJOOR_DB_TIMEOUT=60.0 ``` @@ -291,7 +251,6 @@ export MAJOOR_DB_TIMEOUT=60.0 ### Performance Optimization - Adjust page size based on your system's RAM -- Enable file watching only for actively used directories - Set appropriate timeouts based on your storage speed - Monitor database performance and adjust connections as needed @@ -335,12 +294,6 @@ export MAJOOR_DB_TIMEOUT=60.0 - Ensure tools are properly installed - Test tools independently before using with Assets Manager -#### File Watching Problems -- Check directory permissions -- Verify the watched directories exist and are accessible -- Monitor system resources (file watching can be resource-intensive) -- Adjust intervals if causing performance issues - ### Diagnostic Steps 1. Check ComfyUI console for error messages 2. Verify all required dependencies are installed @@ -364,4 +317,4 @@ export MAJOOR_DB_TIMEOUT=60.0 --- *Settings & Configuration Guide Version: 1.0* -*Last Updated: January 2026* \ No newline at end of file +*Last Updated: January 2026* diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 19bf8a28..f3763571 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -335,18 +335,11 @@ export MAJOOR_FFPROBE_PATH="/path/to/ffprobe" # Set media probe backend export MAJOOR_MEDIA_PROBE_BACKEND="auto" # auto, exiftool, ffprobe, both -# Enable file watcher for auto-reindexing -export MAJOOR_ENABLE_FILE_WATCHER=true - # Database tuning export MAJOOR_DB_TIMEOUT=30.0 export MAJOOR_DB_MAX_CONNECTIONS=8 export MAJOOR_DB_QUERY_TIMEOUT=30.0 -# Watcher settings -export MAJOOR_WATCHER_INTERVAL=15.0 -export MAJOOR_WATCHER_JOIN_TIMEOUT=5.0 - # Collection limits export MJR_COLLECTION_MAX_ITEMS=50000 @@ -372,7 +365,6 @@ If you see warnings about missing dependencies: #### Index Not Updating - Use `Ctrl/Cmd+S` to manually trigger a scan -- Enable file watcher for automatic updates - Check permissions on output directory #### Metadata Extraction Issues @@ -414,4 +406,4 @@ If you see warnings about missing dependencies: --- *Documentation Version: 1.0* -*Last Updated: January 2026* \ No newline at end of file +*Last Updated: January 2026* diff --git a/js/api/client.js b/js/api/client.js index 0116b787..d60ff90c 100644 --- a/js/api/client.js +++ b/js/api/client.js @@ -484,6 +484,22 @@ export async function resetIndex(options = {}) { return post(ENDPOINTS.INDEX_RESET, body); } +export async function setWatcherScope({ scope = "output", customRootId = "" } = {}) { + const s = String(scope || "output").trim().toLowerCase() || "output"; + const rid = String(customRootId || "").trim(); + const body = { scope: s }; + if (rid) body.custom_root_id = rid; + return post(ENDPOINTS.WATCHER_SCOPE, body); +} + +export async function getWatcherStatus() { + return get(ENDPOINTS.WATCHER_STATUS); +} + +export async function toggleWatcher(enabled = true) { + return post(ENDPOINTS.WATCHER_TOGGLE, { enabled: !!enabled }); +} + export async function getToolsStatus(options = {}) { return get(ENDPOINTS.TOOLS_STATUS, options); } diff --git a/js/api/endpoints.js b/js/api/endpoints.js index 7511f1d1..d58cb48b 100644 --- a/js/api/endpoints.js +++ b/js/api/endpoints.js @@ -57,6 +57,11 @@ export const ENDPOINTS = { ASSET_DELETE: "/mjr/am/asset/delete", ASSET_RENAME: "/mjr/am/asset/rename", + // Watcher + WATCHER_STATUS: "/mjr/am/watcher/status", + WATCHER_TOGGLE: "/mjr/am/watcher/toggle", + WATCHER_SCOPE: "/mjr/am/watcher/scope", + // Collections COLLECTIONS: "/mjr/am/collections" }; diff --git a/js/app/i18n.js b/js/app/i18n.js index 557d40f2..a061d521 100644 --- a/js/app/i18n.js +++ b/js/app/i18n.js @@ -18,6 +18,7 @@ const DICTIONARY = { "cat.scanning": "Scanning", "cat.advanced": "Advanced", "cat.security": "Security", + "cat.remote": "Remote Access", // ─── Settings: Grid ─── "setting.grid.minsize.name": "Majoor: Thumbnail Size (px)", @@ -28,8 +29,6 @@ const DICTIONARY = { "setting.sidebar.pos.desc": "Show details sidebar on the left or the right. Reload required.", "setting.siblings.hide.name": "Majoor: Hide PNG Siblings", "setting.siblings.hide.desc": "If a video has a corresponding .png preview, hide the .png from the grid.", - "setting.grid.pagesize.name": "Majoor: Files per Page", - "setting.grid.pagesize.desc": "Number of files loaded per request. More = faster scrolling but more memory.", "setting.nav.infinite.name": "Majoor: Infinite Scroll", "setting.nav.infinite.desc": "Automatically load more files when scrolling.", @@ -42,6 +41,8 @@ const DICTIONARY = { // ─── Settings: Scanning ─── "setting.scan.startup.name": "Majoor: Auto-scan on Startup", "setting.scan.startup.desc": "Start a background scan as soon as ComfyUI loads.", + "setting.watcher.name": "Majoor: File Watcher", + "setting.watcher.desc": "Watch output and custom folders for manually added files and auto-index them in real time.", "setting.sync.rating.name": "Majoor: Sync Rating/Tags to Files", "setting.sync.rating.desc": "Write ratings and tags into file metadata (ExifTool).", @@ -56,6 +57,8 @@ const DICTIONARY = { // ─── Settings: Security ─── "setting.sec.safe.name": "Majoor: Safe Mode", "setting.sec.safe.desc": "When enabled, rating/tags writes are blocked unless explicitly authorized.", + "setting.sec.remote.name": "Majoor: Allow Remote Full Access", + "setting.sec.remote.desc": "Allow non-local clients to perform write operations. Disabling blocks writes unless a token is configured.", "setting.sec.write.name": "Majoor: Allow Write", "setting.sec.write.desc": "Allow writing ratings and tags.", "setting.sec.del.name": "Majoor: Allow Delete", @@ -156,6 +159,10 @@ const DICTIONARY = { "status.withWorkflows": "With workflows: {workflows} - Generation data: {gendata}", "status.lastScan": "Last scan: {date}", "status.scanStats": "Added: {added} - Updated: {updated} - Skipped: {skipped}", + "status.watcher.enabled": "Watcher: enabled", + "status.watcher.enabledScoped": "Watcher: enabled ({scope})", + "status.watcher.disabled": "Watcher: disabled", + "status.watcher.disabledScoped": "Watcher: disabled ({scope})", "status.apiNotFound": "Majoor API endpoints not found (404)", "status.apiNotFoundHint": "Backend routes are not loaded. Restart ComfyUI and check the terminal for Majoor import errors.", "status.errorChecking": "Error checking status", @@ -283,6 +290,7 @@ const DICTIONARY = { "cat.scanning": "Scan", "cat.advanced": "Avancé", "cat.security": "Sécurité", + "cat.remote": "Accès distant", // ─── Settings: Grille ─── "setting.grid.minsize.name": "Majoor: Taille des vignettes (px)", @@ -293,8 +301,6 @@ const DICTIONARY = { "setting.sidebar.pos.desc": "Afficher la sidebar à gauche ou à droite. Recharger la page pour appliquer.", "setting.siblings.hide.name": "Majoor: Masquer aperçus PNG", "setting.siblings.hide.desc": "Si une vidéo a un fichier .png correspondant, masquer le .png de la grille.", - "setting.grid.pagesize.name": "Majoor: Fichiers par page", - "setting.grid.pagesize.desc": "Nombre de fichiers chargés par requête. Plus = plus rapide mais plus de mémoire.", "setting.nav.infinite.name": "Majoor: Défilement infini", "setting.nav.infinite.desc": "Charger automatiquement plus de fichiers en scrollant.", @@ -307,6 +313,8 @@ const DICTIONARY = { // ─── Settings: Scan ─── "setting.scan.startup.name": "Majoor: Auto-scan au démarrage", "setting.scan.startup.desc": "Lancer un scan en arrière-plan dès le chargement de ComfyUI.", + "setting.watcher.name": "Majoor: Surveillance fichiers", + "setting.watcher.desc": "Surveiller les dossiers output et custom pour indexer automatiquement les fichiers ajoutés manuellement.", "setting.sync.rating.name": "Majoor: Sync rating/tags vers fichiers", "setting.sync.rating.desc": "Écrire les notes et tags dans les métadonnées des fichiers (ExifTool).", @@ -321,6 +329,8 @@ const DICTIONARY = { // ─── Settings: Sécurité ─── "setting.sec.safe.name": "Majoor: Mode sécurisé", "setting.sec.safe.desc": "Quand activé, les écritures rating/tags sont bloquées sauf autorisation explicite.", + "setting.sec.remote.name": "Majoor: Autoriser l'accès distant complet", + "setting.sec.remote.desc": "Autorise les clients non locaux à écrire. Désactiver bloque les écritures sauf si un token est configuré.", "setting.sec.write.name": "Majoor: Autoriser écriture", "setting.sec.write.desc": "Autorise l'écriture des notes et tags.", "setting.sec.del.name": "Majoor: Autoriser suppression", @@ -421,6 +431,10 @@ const DICTIONARY = { "status.withWorkflows": "Avec workflows: {workflows} - Données de génération: {gendata}", "status.lastScan": "Dernier scan: {date}", "status.scanStats": "Ajoutés: {added} - Mis à jour: {updated} - Ignorés: {skipped}", + "status.watcher.enabled": "Watcher : activé", + "status.watcher.enabledScoped": "Watcher : activé ({scope})", + "status.watcher.disabled": "Watcher : désactivé", + "status.watcher.disabledScoped": "Watcher : désactivé ({scope})", "status.apiNotFound": "Endpoints API Majoor introuvables (404)", "status.apiNotFoundHint": "Les routes backend ne sont pas chargées. Redémarrez ComfyUI et vérifiez le terminal pour les erreurs d'import Majoor.", "status.errorChecking": "Erreur lors de la vérification", diff --git a/js/app/settings.js b/js/app/settings.js index 1616feff..14a19358 100644 --- a/js/app/settings.js +++ b/js/app/settings.js @@ -3,7 +3,7 @@ */ import { APP_CONFIG, APP_DEFAULTS } from "./config.js"; -import { getSecuritySettings, setSecuritySettings, setProbeBackendMode } from "../api/client.js"; +import { getSecuritySettings, setSecuritySettings, setProbeBackendMode, getWatcherStatus, toggleWatcher } from "../api/client.js"; import { comfyToast } from "./toast.js"; import { safeDispatchCustomEvent } from "../utils/events.js"; import { t, initI18n, setLang, getCurrentLang, getSupportedLanguages } from "./i18n.js"; @@ -45,6 +45,9 @@ const DEFAULT_SETTINGS = { autoScan: { onStartup: APP_DEFAULTS.AUTO_SCAN_ON_STARTUP, }, + watcher: { + enabled: true, + }, status: { pollInterval: APP_DEFAULTS.STATUS_POLL_INTERVAL, }, @@ -90,6 +93,7 @@ const DEFAULT_SETTINGS = { security: { safeMode: false, allowWrite: true, + allowRemoteWrite: true, allowDelete: true, allowRename: true, allowOpenInFolder: true, @@ -145,6 +149,7 @@ export const loadMajoorSettings = () => { "infiniteScroll", "siblings", "autoScan", + "watcher", "status", "viewer", "rtHydrate", @@ -250,6 +255,7 @@ export async function syncBackendSecuritySettings() { settings.security = settings.security || {}; settings.security.safeMode = _safeBool(prefs.safe_mode, settings.security.safeMode); settings.security.allowWrite = _safeBool(prefs.allow_write, settings.security.allowWrite); + settings.security.allowRemoteWrite = _safeBool(prefs.allow_remote_write, settings.security.allowRemoteWrite); settings.security.allowDelete = _safeBool(prefs.allow_delete, settings.security.allowDelete); settings.security.allowRename = _safeBool(prefs.allow_rename, settings.security.allowRename); settings.security.allowOpenInFolder = _safeBool(prefs.allow_open_in_folder, settings.security.allowOpenInFolder); @@ -626,6 +632,34 @@ export const registerMajoorSettings = (app, onApplied) => { }, }); + safeAddSetting({ + id: `${SETTINGS_PREFIX}.Watcher.Enabled`, + category: cat(t("cat.scanning"), t("setting.watcher.name").replace("Majoor: ", "")), + name: t("setting.watcher.name"), + tooltip: t("setting.watcher.desc"), + type: "boolean", + defaultValue: !!settings.watcher?.enabled, + onChange: async (value) => { + settings.watcher = settings.watcher || {}; + settings.watcher.enabled = !!value; + saveMajoorSettings(settings); + notifyApplied("watcher.enabled"); + try { + const res = await toggleWatcher(!!value); + if (!res?.ok) { + settings.watcher.enabled = !value; + saveMajoorSettings(settings); + notifyApplied("watcher.enabled"); + comfyToast(res?.error || "Failed to toggle watcher", "error"); + } + } catch { + settings.watcher.enabled = !value; + saveMajoorSettings(settings); + notifyApplied("watcher.enabled"); + } + }, + }); + safeAddSetting({ id: `${SETTINGS_PREFIX}.RatingTagsSync.Enabled`, category: cat(t("cat.scanning"), t("setting.sync.rating.name").replace("Majoor: ", "")), @@ -695,10 +729,10 @@ export const registerMajoorSettings = (app, onApplied) => { registerMinimapToggle("Enabled", "enabled", "setting.minimap.enabled.name", "setting.minimap.enabled.desc"); - const registerSecurityToggle = (key, nameKey, descKey) => { + const registerSecurityToggle = (key, nameKey, descKey, sectionKey = "cat.security") => { safeAddSetting({ id: `${SETTINGS_PREFIX}.Security.${key}`, - category: cat(t("cat.security"), t(nameKey).replace("Majoor: ", "")), + category: cat(t(sectionKey), t(nameKey).replace("Majoor: ", "")), name: t(nameKey), tooltip: t(descKey), type: "boolean", @@ -713,6 +747,7 @@ export const registerMajoorSettings = (app, onApplied) => { setSecuritySettings({ safe_mode: _safeBool(sec.safeMode, false), allow_write: _safeBool(sec.allowWrite, true), + allow_remote_write: _safeBool(sec.allowRemoteWrite, true), allow_delete: _safeBool(sec.allowDelete, true), allow_rename: _safeBool(sec.allowRename, true), allow_open_in_folder: _safeBool(sec.allowOpenInFolder, true), @@ -732,6 +767,12 @@ export const registerMajoorSettings = (app, onApplied) => { }; registerSecurityToggle("safeMode", "setting.sec.safe.name", "setting.sec.safe.desc"); + registerSecurityToggle( + "allowRemoteWrite", + "setting.sec.remote.name", + "setting.sec.remote.desc", + "cat.remote" + ); registerSecurityToggle("allowWrite", "setting.sec.write.name", "setting.sec.write.desc"); registerSecurityToggle("allowDelete", "setting.sec.del.name", "setting.sec.del.desc"); registerSecurityToggle("allowRename", "setting.sec.ren.name", "setting.sec.ren.desc"); @@ -781,5 +822,30 @@ export const registerMajoorSettings = (app, onApplied) => { } catch {} } + // Best-effort: apply watcher state to backend to match saved settings. + try { + const desired = !!settings?.watcher?.enabled; + setTimeout(() => { + toggleWatcher(desired).catch(() => {}); + }, 0); + } catch {} + return settings; }; + +// Best-effort backend sync (watcher status) at startup. +try { + const settings = loadMajoorSettings(); + if (settings?.watcher && typeof settings.watcher.enabled === "boolean") { + getWatcherStatus() + .then((res) => { + const enabled = !!res?.ok && !!res?.data?.enabled; + if (typeof enabled === "boolean" && enabled !== !!settings.watcher.enabled) { + settings.watcher.enabled = enabled; + saveMajoorSettings(settings); + safeDispatchCustomEvent("mjr-settings-changed", { key: "watcher.enabled" }, { warnPrefix: "[Majoor]" }); + } + }) + .catch(() => {}); + } +} catch {} diff --git a/js/components/Badges.js b/js/components/Badges.js index 1b8ff045..34fee9f0 100644 --- a/js/components/Badges.js +++ b/js/components/Badges.js @@ -138,17 +138,24 @@ export function createWorkflowDot(asset) { const hasGen = toBoolish(asset?.has_generation_data ?? asset?.hasGenerationData); let color = "var(--mjr-status-neutral, #666)"; - let title = "Not parsed yet"; + let title = "Pending: parsing metadata\u2026"; + + const anyTrue = hasWorkflow === true || hasGen === true; + const anyFalse = hasWorkflow === false || hasGen === false; + const anyUnknown = hasWorkflow === null || hasGen === null; if (hasWorkflow === true && hasGen === true) { color = "var(--mjr-status-success, #4CAF50)"; - title = "Complete: Workflow + Generation data"; - } else if (hasWorkflow === true || hasGen === true) { + title = "Complete: workflow + generation data detected"; + } else if (anyTrue) { color = "var(--mjr-status-warning, #FF9800)"; - title = hasWorkflow === true ? "Partial: Workflow only" : "Partial: Generation data only"; - } else if (hasWorkflow === false && hasGen === false) { + title = hasWorkflow === true ? "Partial: workflow only (generation data missing)" : "Partial: generation data only (workflow missing)"; + } else if (anyFalse && !anyTrue && !anyUnknown) { color = "var(--mjr-status-error, #f44336)"; - title = "No workflow or generation data"; + title = "None: no workflow or generation data found"; + } else if (anyUnknown) { + color = "var(--mjr-status-info, #64B5F6)"; + title = "Pending: metadata not parsed yet"; } dot.textContent = "\u25CF"; diff --git a/js/components/Card.js b/js/components/Card.js index a95bebaf..3de628c0 100644 --- a/js/components/Card.js +++ b/js/components/Card.js @@ -17,8 +17,8 @@ import { APP_CONFIG } from "../app/config.js"; * @property {string} [kind] * @property {number} [rating] - 0 to 5 * @property {Array|string|null} [tags] - Array of strings or JSON string - * @property {boolean} [has_workflow] - * @property {boolean} [has_generation_data] + * @property {boolean|null} [has_workflow] + * @property {boolean|null} [has_generation_data] * @property {boolean} [has_unknown_nodes] * @property {number} [width] * @property {number} [height] @@ -419,7 +419,7 @@ const getVideoThumbManager = () => { }; try { - window.addEventListener?.("unload", () => api.dispose()); + window.addEventListener?.("pagehide", () => api.dispose()); window[VIDEO_THUMBS_KEY] = api; } catch {} return api; diff --git a/js/components/ContextMenu.js b/js/components/ContextMenu.js deleted file mode 100644 index 72eaf449..00000000 --- a/js/components/ContextMenu.js +++ /dev/null @@ -1,491 +0,0 @@ -/** - * Context Menu Component - * Right-click menu for asset cards. - * - * Notes: - * - Idempotent: binds once per grid container. - * - Safe defaults: only uses existing backend routes. - * - Uses PrimeIcons (`pi pi-*`) for icons. - */ - -import { getViewerInstance } from "./Viewer.js"; -import { buildAssetViewURL, ENDPOINTS } from "../api/endpoints.js"; -import { comfyToast } from "../app/toast.js"; -import { openInFolder, post } from "../api/client.js"; -import { showAddToCollectionMenu } from "../features/collections/contextmenu/addToCollectionMenu.js"; -import { tryLoadWorkflowToCanvas } from "../features/dnd/DragDrop.js"; -import { stageToInput } from "../features/dnd/staging/stageToInput.js"; -import { pickRootId } from "../utils/ids.js"; -import { MENU_Z_INDEX } from "./contextmenu/MenuCore.js"; -import { app } from "../../scripts/app.js"; - -import { pickBestVideoPathWidget } from "../features/dnd/targets/node.js"; - -// NOTE: Keep this menu isolated from the newer grid context menu -// (`.mjr-grid-context-menu`) to avoid selector collisions. -const MENU_SELECTOR = ".mjr-legacy-context-menu"; - -export function createContextMenu() { - const existing = document.querySelector(MENU_SELECTOR); - if (existing) return existing; - - const menu = document.createElement("div"); - menu.className = "mjr-context-menu mjr-legacy-context-menu"; - menu.style.cssText = ` - position: fixed; - background: var(--comfy-menu-bg); - border: 1px solid var(--border-color); - border-radius: 6px; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); - min-width: 200px; - z-index: ${Number(MENU_Z_INDEX?.MAIN) || 10001}; - display: none; - padding: 4px 0; - `; - - const hide = () => { - menu.style.display = "none"; - }; - - const ac = new AbortController(); - menu._mjrAbortController = ac; - - document.addEventListener( - "click", - (e) => { - if (!menu.contains(e.target)) hide(); - }, - { signal: ac.signal } - ); - document.addEventListener( - "keydown", - (e) => { - if (e.key === "Escape") hide(); - }, - { signal: ac.signal } - ); - document.addEventListener("scroll", hide, { capture: true, passive: true, signal: ac.signal }); - - document.body.appendChild(menu); - return menu; -} - -function createMenuItem(label, iconClass, shortcut, onClick) { - const item = document.createElement("div"); - item.className = "mjr-context-menu-item"; - item.style.cssText = ` - padding: 8px 16px; - display: flex; - align-items: center; - justify-content: space-between; - cursor: pointer; - color: var(--fg-color); - font-size: 13px; - transition: background 0.15s; - user-select: none; - `; - - const leftSide = document.createElement("div"); - leftSide.style.cssText = "display: flex; align-items: center; gap: 10px;"; - - if (iconClass) { - const iconEl = document.createElement("i"); - iconEl.className = iconClass; - iconEl.style.cssText = "width: 16px; text-align: center; opacity: 0.8;"; - leftSide.appendChild(iconEl); - } - - const labelEl = document.createElement("span"); - labelEl.textContent = label; - leftSide.appendChild(labelEl); - item.appendChild(leftSide); - - if (shortcut) { - const shortcutEl = document.createElement("span"); - shortcutEl.textContent = shortcut; - shortcutEl.style.cssText = "font-size: 11px; opacity: 0.5; margin-left: 20px;"; - item.appendChild(shortcutEl); - } - - item.addEventListener("mouseenter", () => { - item.style.background = "var(--comfy-input-bg)"; - }); - item.addEventListener("mouseleave", () => { - item.style.background = "transparent"; - }); - - item.addEventListener("click", (e) => { - e.stopPropagation(); - try { - if (item.dataset?.executing === "1") return; - item.dataset.executing = "1"; - } catch {} - - try { - const res = onClick?.(); - - // Auto-close menu if logical - try { - // If it returns a promise, wait for it - if (res && typeof res.then === "function") { - res.finally(() => { - try { item.dataset.executing = "0"; } catch {} - try { - const parentMenu = item.closest(".mjr-legacy-context-menu"); - if (parentMenu) parentMenu.style.display = "none"; - } catch {} - }); - return; - } - - // Immediate close for sync actions - try { - const parentMenu = item.closest(".mjr-legacy-context-menu"); - if (parentMenu) parentMenu.style.display = "none"; - } catch {} - } catch {} - - try { - item.dataset.executing = "0"; - } catch {} - } catch (err) { - console.error("[LegacyContextMenu] Action failed:", err); - try { - item.dataset.executing = "0"; - } catch {} - } - }); - - return item; -} - -function createMenuSeparator() { - const separator = document.createElement("div"); - separator.className = "mjr-context-menu-separator"; - separator.style.cssText = ` - height: 1px; - background: var(--border-color); - margin: 4px 0; - `; - return separator; -} - -export function showContextMenu(x, y, asset, allAssets, currentIndex, selectedAssets = [], { onDetails } = {}) { - const menu = createContextMenu(); - menu.innerHTML = ""; - - const selectedCount = selectedAssets.length; - const hide = () => { - try { - menu.style.display = "none"; - } catch {} - }; - - const openViewer = createMenuItem("Open in Viewer", "pi pi-eye", "Double-click", () => { - const viewer = getViewerInstance(); - if (selectedCount >= 2 && selectedCount <= 4) { - viewer.open(selectedAssets, 0); - } else { - viewer.open(allAssets, currentIndex); - } - }); - menu.appendChild(openViewer); - - // --- WORKFLOW ACTIONS --- - if (asset.has_workflow || asset.has_generation_data) { - // [CustomScripts Feature] Load Workflow directly - const loadWf = createMenuItem("Load Workflow", "pi pi-sitemap", null, async () => { - const payload = { - filename: asset.filename, - subfolder: asset.subfolder, - type: asset.type, - root_id: pickRootId(asset) - }; - - // Only confirm if graph is not empty (basic heuristic) - if (app.graph && app.graph._nodes && app.graph._nodes.length > 0) { - if (!confirm("⚠️ Replace current workflow?\n\nUnsaved changes will be lost.")) return; - } - - hide(); - const ok = await tryLoadWorkflowToCanvas(payload); - if (ok) { - comfyToast("Workflow loaded", "success"); - } else { - comfyToast("Could not load workflow (missing metadata?)", "warn"); - } - }); - menu.appendChild(loadWf); - } - - // --- SEND TO NODE --- - const loadImageNodes = app?.graph?._nodes?.filter(n => { - if (!n?.type) return false; - const t = n.type.toLowerCase(); - // Broad matching for Loaders - return (t.includes("load") || t.includes("input")) && (t.includes("image") || t.includes("video")); - }) || []; - - if (loadImageNodes.length > 0) { - // [CustomScripts Feature] Send to LoadImage - const sendToInput = createMenuItem("Send to Node", "pi pi-arrow-circle-down", null, async () => { - hide(); - - // 1. Prefer selected node - // 2. Prefer node matching the file type (video vs image) if multiple - // 3. Fallback to first - let targetNode = app.canvas.current_node; - - const ext = String(asset.filename).split('.').pop(); - const isVideo = ["mp4", "webm", "mkv", "mov", "gif"].includes(ext?.toLowerCase()); - - if (!targetNode || !loadImageNodes.includes(targetNode)) { - if (isVideo) { - targetNode = loadImageNodes.find(n => n.type.toLowerCase().includes("video")) || loadImageNodes[0]; - } else { - targetNode = loadImageNodes.find(n => !n.type.toLowerCase().includes("video")) || loadImageNodes[0]; - } - } - - if (!targetNode) return; - - const payload = { - filename: asset.filename, - subfolder: asset.subfolder, - type: asset.type, - root_id: pickRootId(asset) - }; - - const relativePath = await stageToInput({ - post, - endpoint: ENDPOINTS.STAGE_TO_INPUT, - payload, - index: false - }); - - if (relativePath) { - const widget = pickBestVideoPathWidget(targetNode, ext) || targetNode.widgets?.find(w => w.name === "image" || w.name === "video" || w.name === "filename"); - - if (widget) { - widget.value = relativePath; - if (widget.callback) widget.callback(widget.value); - - // Flash the node to show where it went - app.canvas.centerOnNode(targetNode); - app.canvas.draw(true, true); - - comfyToast(`Sent to ${targetNode.title || "Node"}`, "success"); - } else { - // Fallback: set first string widget - const firstStr = targetNode.widgets?.find(w => typeof w.value === "string" || w.type === "string"); - if (firstStr) { - firstStr.value = relativePath; - if (firstStr.callback) firstStr.callback(firstStr.value); - comfyToast(`Sent to ${targetNode.title} (guessed widget)`, "success"); - } else { - comfyToast(`Node ${targetNode.type} has no file widget`, "warn"); - } - } - app.graph.setDirtyCanvas(true); - } - }); - menu.appendChild(sendToInput); - } - - menu.appendChild(createMenuSeparator()); - - const details = createMenuItem("Details", "pi pi-info-circle", "D", () => { - hide(); - try { - onDetails?.(asset); - } catch {} - }); - menu.appendChild(details); - - const openNewTab = createMenuItem("Open in New Tab", "pi pi-external-link", null, () => { - hide(); - const url = buildAssetViewURL(asset); - window.open(url, "_blank"); - }); - menu.appendChild(openNewTab); - - menu.appendChild(createMenuSeparator()); - - if (selectedCount === 2) { - const ab = createMenuItem("Compare A/B (2 selected)", "pi pi-clone", null, () => { - const viewer = getViewerInstance(); - viewer.open(selectedAssets, 0); - viewer.setMode("ab"); - }); - menu.appendChild(ab); - } - - if (selectedCount >= 2 && selectedCount <= 4) { - const side = createMenuItem(`Side-by-side (2×2) (${selectedCount} selected)`, "pi pi-table", null, () => { - const viewer = getViewerInstance(); - viewer.open(selectedAssets, 0); - viewer.setMode("sidebyside"); - }); - menu.appendChild(side); - } - - const copyPath = createMenuItem("Copy file path", "pi pi-copy", null, async () => { - try { - const path = asset?.filepath ? String(asset.filepath) : ""; - if (!path) { - comfyToast("No file path available for this asset.", "error"); - return; - } - await navigator.clipboard.writeText(path); - comfyToast("File path copied to clipboard", "success", 2000); - } catch (error) { - console.error("Copy path failed:", error); - comfyToast("Failed to copy path", "error"); - } - hide(); - }); - menu.appendChild(copyPath); - - const addToCollection = createMenuItem( - selectedCount > 1 ? `Add to collection... (${selectedCount})` : "Add to collection...", - "pi pi-bookmark", - null, - async () => { - hide(); - try { - const list = selectedAssets?.length ? selectedAssets : [asset]; - await showAddToCollectionMenu({ x, y, assets: list }); - } catch (err) { - console.error("Add to collection failed:", err); - } - } - ); - menu.appendChild(addToCollection); - - const openFolder = createMenuItem("Open in Folder", "pi pi-folder-open", null, async () => { - try { - const assetId = asset?.id; - if (!assetId) { - comfyToast("No asset id available for this item.", "error"); - return; - } - const res = await openInFolder(assetId); - if (!res?.ok) { - comfyToast(res?.error || "Failed to open folder.", "error"); - } else { - comfyToast("Opened in folder", "info", 2000); - } - } catch (error) { - console.error("Open in folder failed:", error); - comfyToast("Failed to open folder", "error"); - } - hide(); - }); - menu.appendChild(openFolder); - - menu.style.display = "block"; - - const rect = menu.getBoundingClientRect(); - const viewportWidth = window.innerWidth; - const viewportHeight = window.innerHeight; - - let finalX = x; - let finalY = y; - - if (x + rect.width > viewportWidth) { - finalX = viewportWidth - rect.width - 10; - } - if (y + rect.height > viewportHeight) { - finalY = viewportHeight - rect.height - 10; - } - - menu.style.left = `${finalX}px`; - menu.style.top = `${finalY}px`; -} - -export function bindContextMenu(gridContainer, assets) { - try { - gridContainer._mjrContextMenuAssets = Array.isArray(assets) ? assets : []; - } catch {} - - // If the panel has the newer grid context menu bound, don't bind the legacy one. - try { - if (gridContainer?._mjrGridContextMenuBound) return; - } catch {} - - if (gridContainer._mjrContextMenuBound && typeof gridContainer._mjrContextMenuUnbind === "function") { - return gridContainer._mjrContextMenuUnbind; - } - gridContainer._mjrContextMenuBound = true; - - const handler = (e) => { - const card = e.target.closest(".mjr-asset-card"); - if (!card) return; - - e.preventDefault(); - e.stopPropagation(); - - // If right-clicking on a non-selected card, treat it as a single selection. - try { - if (!card.classList.contains("is-selected")) { - for (const el of gridContainer.querySelectorAll(".mjr-asset-card.is-selected")) { - el.classList.remove("is-selected"); - el.setAttribute?.("aria-selected", "false"); - } - card.classList.add("is-selected"); - card.setAttribute?.("aria-selected", "true"); - try { - const id = card?.dataset?.mjrAssetId; - if (id) { - gridContainer.dataset.mjrSelectedAssetId = String(id); - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify([String(id)]); - } - } catch {} - } - } catch {} - - const currentAsset = card._mjrAsset; - if (!currentAsset) return; - - const list = gridContainer._mjrContextMenuAssets || []; - const idx = Array.isArray(list) ? list.findIndex((a) => a.id === currentAsset.id) : -1; - - const selected = Array.from(gridContainer.querySelectorAll(".mjr-asset-card.is-selected")) - .map((c) => c?._mjrAsset) - .filter(Boolean); - - // Keep dataset selection in sync so selection survives grid re-renders. - try { - const ids = Array.from(gridContainer.querySelectorAll(".mjr-asset-card.is-selected")) - .map((c) => c?.dataset?.mjrAssetId) - .filter(Boolean) - .map(String); - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(ids); - if (card?.dataset?.mjrAssetId) gridContainer.dataset.mjrSelectedAssetId = String(card.dataset.mjrAssetId); - } catch {} - - showContextMenu(e.clientX, e.clientY, currentAsset, list, idx, selected, { - onDetails: typeof gridContainer._mjrOpenDetails === "function" ? gridContainer._mjrOpenDetails : null, - }); - }; - - try { - gridContainer.addEventListener("contextmenu", handler); - } catch (err) { - console.error("[LegacyContextMenu] Failed to bind:", err); - } - - const unbind = () => { - try { - gridContainer.removeEventListener("contextmenu", handler); - } catch {} - try { - gridContainer._mjrContextMenuBound = false; - } catch {} - try { - gridContainer._mjrContextMenuUnbind = null; - } catch {} - }; - gridContainer._mjrContextMenuUnbind = unbind; - return unbind; -} diff --git a/js/components/Viewer.js b/js/components/Viewer.js index f0bc207d..a583c5f5 100644 --- a/js/components/Viewer.js +++ b/js/components/Viewer.js @@ -10,6 +10,7 @@ import { bindViewerContextMenu } from "../features/viewer/ViewerContextMenu.js"; import { createFileBadge, createRatingBadge, createTagsBadge } from "./Badges.js"; import { APP_CONFIG } from "../app/config.js"; import { safeDispatchCustomEvent } from "../utils/events.js"; +import { safeClosest } from "../utils/dom.js"; import { mountVideoControls } from "./VideoControls.js"; import { createDefaultViewerState } from "../features/viewer/state.js"; import { createViewerLifecycle, destroyMediaProcessorsIn, safeAddListener, safeCall } from "../features/viewer/lifecycle.js"; @@ -590,16 +591,16 @@ export function createViewer() { } const t = e.target; - if (t.closest(".mjr-viewer-header") || - t.closest(".mjr-viewer-footer") || - t.closest(".mjr-viewer-geninfo") || - t.closest(".mjr-video-controls") || - t.closest(".mjr-context-menu") || - t.closest(".mjr-ab-slider") || - t.closest(".mjr-viewer-loupe") || - t.closest(".mjr-viewer-probe") || - t.closest(".mjr-viewer-media") || - t.tagName === "IMG" || t.tagName === "VIDEO" || t.tagName === "CANVAS") { + if (safeClosest(t, ".mjr-viewer-header") || + safeClosest(t, ".mjr-viewer-footer") || + safeClosest(t, ".mjr-viewer-geninfo") || + safeClosest(t, ".mjr-video-controls") || + safeClosest(t, ".mjr-context-menu") || + safeClosest(t, ".mjr-ab-slider") || + safeClosest(t, ".mjr-viewer-loupe") || + safeClosest(t, ".mjr-viewer-probe") || + safeClosest(t, ".mjr-viewer-media") || + (t && (t.tagName === "IMG" || t.tagName === "VIDEO" || t.tagName === "CANVAS"))) { return; } _requestCloseFromButton(); diff --git a/js/entry.js b/js/entry.js index 59a80e8e..962401fc 100644 --- a/js/entry.js +++ b/js/entry.js @@ -8,145 +8,84 @@ import { testAPI, triggerStartupScan } from "./app/bootstrap.js"; import { ensureStyleLoaded } from "./app/style.js"; import { registerMajoorSettings } from "./app/settings.js"; import { initDragDrop } from "./features/dnd/DragDrop.js"; -import { loadAssets, upsertAsset, captureAnchor, restoreAnchor, isAtBottom, scrollToBottom, flushUpsertBatch } from "./features/grid/GridView.js"; +import { loadAssets, upsertAsset } from "./features/grid/GridView.js"; import { renderAssetsManager, getActiveGridContainer } from "./features/panel/AssetsManagerPanel.js"; import { extractOutputFiles } from "./utils/extractOutputFiles.js"; -import { post, get } from "./api/client.js"; +import { post } from "./api/client.js"; import { ENDPOINTS } from "./api/endpoints.js"; -import { buildListURL } from "./api/endpoints.js"; const UI_FLAGS = { useComfyThemeUI: true, }; -const ENTRY_RUNTIME_KEY = "__MJR_AM_ENTRY_RUNTIME__"; -const UI_BOOT_DELAY_MS = 1500; -const RAF_STABILIZE_FRAMES = 2; -const AUTO_REFRESH_DEBOUNCE_MS = 500; -const AUTO_INSERT_FETCH_LIMIT = 50; - -// Queue limits to prevent memory exhaustion and UI freeze -const PENDING_FILES_MAX = 200; // Max files in queue before dropping oldest -const EXECUTION_STARTS_MAX = 50; // Max tracked prompt IDs -const EXECUTION_STARTS_TTL_MS = 300000; // 5 min TTL for orphan cleanup -const PROCESS_QUEUE_DEBOUNCE_MS = 100; // Debounce queue processing -const PROCESS_QUEUE_BATCH_MAX = 20; // Max files per batch to avoid long blocking - -const _raf = () => - new Promise((resolve) => { +// Runtime cleanup key (used for hot-reload cleanup in ComfyUI) +const ENTRY_RUNTIME_KEY = "__MJR_ENTRY_RUNTIME__"; +const ENTRY_ABORT = typeof AbortController !== "undefined" ? new AbortController() : null; +try { + if (typeof window !== "undefined") { + // Replace previous runtime entry to allow cleanup on hot-reload try { - requestAnimationFrame(() => resolve(true)); - } catch { - resolve(false); - } - }); - -function isObservabilityEnabled() { - try { - const raw = localStorage?.getItem?.("mjrSettings"); - if (!raw) return false; - const parsed = JSON.parse(raw); - return !!parsed?.observability?.enabled; - } catch { - return false; + const prev = window[ENTRY_RUNTIME_KEY]; + if (prev && prev.controller && typeof prev.controller.abort === "function") { + try { prev.controller.abort(); } catch {} + } + } catch {} + window[ENTRY_RUNTIME_KEY] = { controller: ENTRY_ABORT }; } +} catch {} + +// Deduplication for executed events (ComfyUI can fire multiple times for same file) +const DEDUPE_TTL_MS = 2000; +const recentFiles = new Map(); // key -> timestamp + +// Track execution start times to compute generation duration +const EXECUTION_START_TTL_MS = 10 * 60 * 1000; +const executionStarts = new Map(); // prompt_id -> timestamp(ms) + +function rememberExecutionStart(promptId, timestampMs) { + if (!promptId) return; + const ts = Number(timestampMs) || Date.now(); + executionStarts.set(String(promptId), ts); + pruneExecutionStarts(); +} + +function pruneExecutionStarts() { + const now = Date.now(); + for (const [key, ts] of executionStarts) { + if (!ts || now - ts > EXECUTION_START_TTL_MS) { + executionStarts.delete(key); + } + } +} + +function dedupeFiles(files) { + const now = Date.now(); + // Prune old entries + for (const [key, ts] of recentFiles) { + if (now - ts > DEDUPE_TTL_MS) recentFiles.delete(key); + } + // Filter out recently seen files + const fresh = []; + for (const f of files) { + const key = `${f.type || ""}|${f.subfolder || ""}|${f.filename || ""}`.toLowerCase(); + if (!recentFiles.has(key)) { + recentFiles.set(key, now); + fresh.push(f); + } + } + return fresh; } app.registerExtension({ name: "Majoor.AssetsManager", async setup() { - // Best-effort hot-reload safety: abort previous listeners/timers for this extension instance. - const runtime = (() => { - try { - const w = typeof window !== "undefined" ? window : null; - const prev = w?.[ENTRY_RUNTIME_KEY] || null; - try { - prev?.abort?.(); - } catch {} - - const ac = typeof AbortController !== "undefined" ? new AbortController() : null; - const timers = new Set(); - const trackTimeout = (fn, ms) => { - try { - const id = setTimeout(fn, ms); - timers.add(id); - return id; - } catch { - return null; - } - }; - const clearTimers = () => { - try { - for (const t of Array.from(timers)) { - try { - clearTimeout(t); - } catch {} - } - } catch {} - try { - timers.clear(); - } catch {} - }; - try { - ac?.signal?.addEventListener?.("abort", clearTimers, { once: true }); - } catch {} - - const delay = (ms) => - new Promise((resolve) => { - try { - if (ac?.signal?.aborted) return resolve(false); - } catch {} - const id = trackTimeout(() => resolve(true), ms); - try { - ac?.signal?.addEventListener?.( - "abort", - () => { - try { - if (id != null) clearTimeout(id); - } catch {} - resolve(false); - }, - { once: true } - ); - } catch {} - }); - - const cleanupCbs = []; - - const rt = { - ac, - trackTimeout, - delay, - register: (fn) => { if (typeof fn === "function") cleanupCbs.push(fn); }, - abort: () => { - try { - clearTimers(); - } catch {} - try { - ac?.abort?.(); - } catch {} - try { - cleanupCbs.forEach(fn => { try { fn(); } catch {} }); - } catch {} - }, - }; - - try { - if (w) w[ENTRY_RUNTIME_KEY] = rt; - } catch {} - return rt; - } catch { - return { ac: null, trackTimeout: setTimeout, delay: (ms) => new Promise((r) => setTimeout(() => r(true), ms)), abort: () => {} }; - } - })(); - + // Initialize core services testAPI(); ensureStyleLoaded({ enabled: UI_FLAGS.useComfyThemeUI }); - + try { - const disposeDnD = initDragDrop(); - runtime.register(disposeDnD); + initDragDrop(); } catch {} registerMajoorSettings(app, () => { @@ -156,261 +95,100 @@ app.registerExtension({ triggerStartupScan(); + // Get ComfyUI API const api = app.api || (app.ui ? app.ui.api : null); if (api) { - // [ISSUE 1] Backend Scan Notification -> UI Event + // Clean up previous handlers (hot-reload safety) try { - api.addEventListener("mjr-scan-complete", (e) => { - const detail = e.detail; - window.dispatchEvent(new CustomEvent('mjr:scan-complete', { detail })); - }); - } catch (err) { - console.warn("[Majoor] Failed to bind scan listener", err); - } - - // Avoid double-binding when ComfyUI reloads extensions. - try { - if (api._mjrRtHandlers && typeof api._mjrRtHandlers === "object") { - for (const [evt, handler] of Object.entries(api._mjrRtHandlers)) { - try { - api.removeEventListener(String(evt), handler); - } catch {} - } - } else if (api._mjrExecutedHandler) { + if (api._mjrExecutedHandler) { api.removeEventListener("executed", api._mjrExecutedHandler); } + if (api._mjrAssetAddedHandler) { + api.removeEventListener("mjr-asset-added", api._mjrAssetAddedHandler); + } + if (api._mjrAssetUpdatedHandler) { + api.removeEventListener("mjr-asset-updated", api._mjrAssetUpdatedHandler); + } + if (api._mjrScanCompleteHandler) { + api.removeEventListener("mjr-scan-complete", api._mjrScanCompleteHandler); + } + if (api._mjrExecutionStartHandler) { + api.removeEventListener("execution_start", api._mjrExecutionStartHandler); + } + if (api._mjrExecutionEndHandler) { + api.removeEventListener("execution_success", api._mjrExecutionEndHandler); + api.removeEventListener("execution_error", api._mjrExecutionEndHandler); + api.removeEventListener("execution_interrupted", api._mjrExecutionEndHandler); + } } catch {} - // Batch state for debouncing multiple checks - api._mjrPendingFiles = api._mjrPendingFiles || []; - api._mjrProcessingQueue = false; // Mutex for async processing - api._mjrExecutionStarts = api._mjrExecutionStarts || new Map(); - api._mjrProcessQueueTimer = null; // Debounce timer - api._mjrLastQueueProcess = 0; // Last process timestamp + // Listen for ComfyUI execution - extract output files and send to backend + api._mjrExecutedHandler = (event) => { + const outputFiles = dedupeFiles(extractOutputFiles(event?.detail?.output)); + if (!outputFiles.length) return; - // Cleanup old execution starts to prevent memory leak - const cleanupExecutionStarts = () => { - try { - if (!api._mjrExecutionStarts || api._mjrExecutionStarts.size === 0) return; - const now = Date.now(); - const toDelete = []; - for (const [id, ts] of api._mjrExecutionStarts.entries()) { - if (now - ts > EXECUTION_STARTS_TTL_MS) { - toDelete.push(id); - } - } - for (const id of toDelete) { - api._mjrExecutionStarts.delete(id); - } - // Also limit size - while (api._mjrExecutionStarts.size > EXECUTION_STARTS_MAX) { - const oldest = api._mjrExecutionStarts.keys().next().value; - if (oldest) api._mjrExecutionStarts.delete(oldest); - } - } catch {} + const promptId = event?.detail?.prompt_id || event?.detail?.promptId; + const startTs = promptId ? executionStarts.get(String(promptId)) : null; + const genTimeMs = startTs ? Math.max(0, Date.now() - startTs) : 0; + + const files = genTimeMs > 0 + ? outputFiles.map((f) => ({ ...f, generation_time_ms: genTimeMs })) + : outputFiles; + + post(ENDPOINTS.INDEX_FILES, { files, origin: "generation" }).catch(() => {}); }; + api.addEventListener("executed", api._mjrExecutedHandler); - // Listen for execution start to effectively measure generation time - const onExecutionStart = (event) => { - const promptId = event?.detail?.prompt_id; + // Track execution start/end to compute duration + api._mjrExecutionStartHandler = (event) => { + const promptId = event?.detail?.prompt_id || event?.detail?.promptId; + const ts = event?.detail?.timestamp; + rememberExecutionStart(promptId, ts); + }; + api._mjrExecutionEndHandler = (event) => { + const promptId = event?.detail?.prompt_id || event?.detail?.promptId; if (promptId) { - api._mjrExecutionStarts.set(promptId, Date.now()); - cleanupExecutionStarts(); + executionStarts.delete(String(promptId)); } + pruneExecutionStarts(); }; - api.addEventListener("execution_start", onExecutionStart); - - // Async queue processor that mimics "Image Feed" reactivity but ensures DB consistency - const processFileQueue = async () => { - if (api._mjrProcessingQueue) return; // Already working - if (api._mjrPendingFiles.length === 0) return; - - api._mjrProcessingQueue = true; - api._mjrLastQueueProcess = Date.now(); - - // Grab current batch (limited size to avoid blocking UI) - const batchSize = Math.min(api._mjrPendingFiles.length, PROCESS_QUEUE_BATCH_MAX); - const batch = api._mjrPendingFiles.splice(0, batchSize); - - try { - // 1. Index the files (backend) - // console.log(`📂 Majoor [ℹ️] Indexing batch of ${batch.length} files...`); - const indexRes = await post(ENDPOINTS.INDEX_FILES, { files: batch, incremental: true }); - - if (!indexRes?.ok) { - console.warn("📂 Majoor [⚠️] index-files failed:", indexRes?.error || indexRes); - } - - // 2. Fetch latest & visual update - // Only update if the user is looking at the default view (no search query) - const latestGrid = getActiveGridContainer(); - const currentQuery = String(latestGrid?.dataset?.mjrQuery || "*").trim() || "*"; - - if (latestGrid && (currentQuery === "*" || currentQuery === "")) { - // Use incremental update logic - const anchor = captureAnchor(latestGrid); - const atBottomBefore = isAtBottom(latestGrid); - - // Read filter state from DOM - const scope = latestGrid.dataset?.mjrScope || "output"; - const customRootId = latestGrid.dataset?.mjrCustomRootId || ""; - const subfolder = latestGrid.dataset?.mjrSubfolder || ""; - const kind = latestGrid.dataset?.mjrFilterKind || ""; - const workflowOnly = latestGrid.dataset?.mjrFilterWorkflowOnly === "1"; - const minRating = Number(latestGrid.dataset?.mjrFilterMinRating || 0) || 0; - const dateRange = String(latestGrid.dataset?.mjrFilterDateRange || "").trim().toLowerCase(); - const dateExact = String(latestGrid.dataset?.mjrFilterDateExact || "").trim(); - - // Fetch the newest page (Limit increased to catch batch bursts) - // This "rescans from 0" (offset 0) to get the latest - const url = buildListURL({ - q: "*", - limit: Math.max(AUTO_INSERT_FETCH_LIMIT, batch.length * 2), // Ensure we fetch enough to cover the batch - offset: 0, - scope, - subfolder, - customRootId: customRootId || null, - kind: kind || null, - hasWorkflow: workflowOnly ? true : null, - minRating: minRating > 0 ? minRating : null, - dateRange: dateRange || null, - dateExact: dateExact || null, - includeTotal: false - }); - - const result = await get(url); - if (result?.ok && Array.isArray(result.data?.assets)) { - // Batch upserts to avoid multiple setItems calls - for (const asset of result.data.assets) { - try { - upsertAsset(latestGrid, asset); - } catch {} - } - // Flush the batch before scroll operations - flushUpsertBatch(latestGrid); - } - - // Auto-scroll logic - if (atBottomBefore) { - for (let i = 0; i < RAF_STABILIZE_FRAMES; i++) await _raf(); - scrollToBottom(latestGrid); - } else { - await restoreAnchor(latestGrid, anchor); - } - } - - } catch (err) { - console.error("📂 Majoor [❌] Queue processing error:", err); - } finally { - api._mjrProcessingQueue = false; - - // If new files arrived while we were working, schedule next batch with debounce - if (api._mjrPendingFiles.length > 0) { - // Yield to UI thread with debounce to prevent tight loops - if (api._mjrProcessQueueTimer) clearTimeout(api._mjrProcessQueueTimer); - api._mjrProcessQueueTimer = setTimeout(() => { - api._mjrProcessQueueTimer = null; - processFileQueue(); - }, PROCESS_QUEUE_DEBOUNCE_MS); - } - } - }; - - // Debounced queue trigger - const scheduleProcessQueue = () => { - if (api._mjrProcessQueueTimer) return; // Already scheduled - if (api._mjrProcessingQueue) return; // Already processing - - api._mjrProcessQueueTimer = setTimeout(() => { - api._mjrProcessQueueTimer = null; - processFileQueue(); - }, PROCESS_QUEUE_DEBOUNCE_MS); - }; - - api._mjrExecutedHandler = async (event) => { - try { - if (runtime?.ac?.signal?.aborted) return; - } catch {} - - const detail = event?.detail; - const output = detail?.output || detail || null; - const promptId = detail?.prompt_id; + api.addEventListener("execution_start", api._mjrExecutionStartHandler); + api.addEventListener("execution_success", api._mjrExecutionEndHandler); + api.addEventListener("execution_error", api._mjrExecutionEndHandler); + api.addEventListener("execution_interrupted", api._mjrExecutionEndHandler); + // Listen for backend push - upsert asset directly to grid + api._mjrAssetAddedHandler = (event) => { const grid = getActiveGridContainer(); - if (!output) return; - - const hasMedia = output.images || output.gifs || output.videos; - if (!hasMedia) return; - - const newFiles = extractOutputFiles(output); - if (!newFiles.length) return; - - // Calculate generation time if we saw the start - let genTimeMs = undefined; - if (promptId && api._mjrExecutionStarts.has(promptId)) { - const start = api._mjrExecutionStarts.get(promptId); - genTimeMs = Date.now() - start; - api._mjrExecutionStarts.delete(promptId); // Cleanup + if (grid && event?.detail) { + upsertAsset(grid, event.detail); } - - if (genTimeMs) { - // Attach duration to each file object so backend can index it - for (const f of newFiles) { - f.generation_time_ms = genTimeMs; - } - } - - // Add to batch with size limit - api._mjrPendingFiles.push(...newFiles); - - // Enforce queue size limit (drop oldest if overflow) - while (api._mjrPendingFiles.length > PENDING_FILES_MAX) { - api._mjrPendingFiles.shift(); - } - - // Trigger processing with debounce - scheduleProcessQueue(); }; + api.addEventListener("mjr-asset-added", api._mjrAssetAddedHandler); - // Bind to multiple event names across ComfyUI variants. - const rtHandlers = {}; - const bind = (evt) => { + api._mjrAssetUpdatedHandler = (event) => { + const grid = getActiveGridContainer(); + if (grid && event?.detail) { + upsertAsset(grid, event.detail); + } + }; + api.addEventListener("mjr-asset-updated", api._mjrAssetUpdatedHandler); + + // Listen for scan-complete events to avoid "Unknown message type" warnings + api._mjrScanCompleteHandler = (event) => { try { - api.addEventListener(evt, api._mjrExecutedHandler, runtime?.ac ? { signal: runtime.ac.signal } : undefined); - rtHandlers[evt] = api._mjrExecutedHandler; + const detail = event?.detail || {}; + window.dispatchEvent(new CustomEvent("mjr-scan-complete", { detail })); } catch {} }; - for (const evt of ["executed", "execution_success", "execution_complete"]) bind(evt); - api._mjrRtHandlers = rtHandlers; - try { - runtime?.ac?.signal?.addEventListener?.( - "abort", - () => { - try { - if (api._mjrRtHandlers && typeof api._mjrRtHandlers === "object") { - for (const [evt, handler] of Object.entries(api._mjrRtHandlers)) { - try { - api.removeEventListener(String(evt), handler); - } catch {} - } - } - } catch {} - try { - api._mjrRtHandlers = {}; - } catch {} - try { - api._mjrExecutedHandler = null; - } catch {} - }, - { once: true } - ); - } catch {} + api.addEventListener("mjr-scan-complete", api._mjrScanCompleteHandler); console.log("📂 Majoor [✅] Real-time listener registered"); } else { console.warn("📂 Majoor [⚠️] API not available, real-time updates disabled"); } + // Register sidebar tab if (app.extensionManager?.registerSidebarTab) { app.extensionManager.registerSidebarTab({ id: "majoor-assets", diff --git a/js/features/contextmenu/GridContextMenu.js b/js/features/contextmenu/GridContextMenu.js index 2a3984bf..1343c0d0 100644 --- a/js/features/contextmenu/GridContextMenu.js +++ b/js/features/contextmenu/GridContextMenu.js @@ -10,7 +10,9 @@ import { ASSET_RATING_CHANGED_EVENT, ASSET_TAGS_CHANGED_EVENT } from "../../app/ import { createTagsEditor } from "../../components/TagsEditor.js"; import { getViewerInstance } from "../../components/Viewer.js"; import { safeDispatchCustomEvent } from "../../utils/events.js"; +import { safeClosest } from "../../utils/dom.js"; import { showAddToCollectionMenu } from "../collections/contextmenu/addToCollectionMenu.js"; +import { removeAssetsFromGrid } from "../grid/GridView.js"; import { getOrCreateMenu, createMenuItem, @@ -48,6 +50,21 @@ const getSelectedAssetIds = (gridContainer) => { const getSelectedAssets = (gridContainer) => { const assets = []; if (!gridContainer) return assets; + try { + const raw = gridContainer.dataset?.mjrSelectedAssetIds; + if (raw) { + const ids = JSON.parse(raw); + if (Array.isArray(ids) && ids.length) { + const all = typeof gridContainer?._mjrGetAssets === "function" ? gridContainer._mjrGetAssets() : []; + const idSet = new Set(ids.map(String)); + for (const a of all || []) { + const id = String(a?.id || ""); + if (id && idSet.has(id)) assets.push(a); + } + if (assets.length) return assets; + } + } + } catch {} try { const cards = gridContainer.querySelectorAll(".mjr-asset-card.is-selected"); for (const card of cards) { @@ -70,12 +87,25 @@ const getAssetFilepath = (asset) => { const clearSelection = (gridContainer) => { if (!gridContainer) return; try { + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection([], ""); + return; + } const cards = gridContainer.querySelectorAll('.mjr-asset-card.is-selected'); for (const card of cards) { card.classList.remove('is-selected'); card.setAttribute('aria-selected', 'false'); } } catch {} + try { + delete gridContainer.dataset.mjrSelectedAssetIds; + delete gridContainer.dataset.mjrSelectedAssetId; + } catch {} + safeDispatchCustomEvent( + "mjr:selection-changed", + { selectedIds: [] }, + { target: gridContainer, warnPrefix: "[GridContextMenu]" } + ); }; const selectCardForDetails = (gridContainer, card, asset, state) => { @@ -92,8 +122,12 @@ const selectCardForDetails = (gridContainer, card, asset, state) => { const id = asset?.id != null ? String(asset.id) : ""; if (id) { try { - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify([id]); - gridContainer.dataset.mjrSelectedAssetId = id; + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection([id], id); + } else { + gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify([id]); + gridContainer.dataset.mjrSelectedAssetId = id; + } } catch {} } else { try { @@ -116,6 +150,13 @@ const selectCardForDetails = (gridContainer, card, asset, state) => { const getAllAssetsInGrid = (gridContainer) => { const assets = []; if (!gridContainer) return assets; + try { + const getter = gridContainer._mjrGetAssets; + if (typeof getter === "function") { + const list = getter(); + if (Array.isArray(list) && list.length) return list; + } + } catch {} try { const cards = gridContainer.querySelectorAll(".mjr-asset-card"); for (const card of cards) { @@ -303,7 +344,7 @@ export function bindGridContextMenu({ const handler = async (e) => { // Check if the contextmenu event is on an asset card - const card = e.target.closest(".mjr-asset-card"); + const card = safeClosest(e.target, ".mjr-asset-card"); if (!card) return; e.preventDefault(); @@ -329,12 +370,17 @@ export function bindGridContextMenu({ if (asset && asset.id && !selectedIds.has(String(asset.id))) { const newSelection = [String(asset.id)]; try { - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(newSelection); - // Dispatch event to sync state - safeDispatchCustomEvent("mjr:selection-changed", { - selectedIds: newSelection, - selectedAssets: [asset] - }, { target: gridContainer, warnPrefix: "[GridContextMenu]" }); + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection(newSelection, newSelection[0]); + } else { + gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(newSelection); + gridContainer.dataset.mjrSelectedAssetId = newSelection[0]; + safeDispatchCustomEvent("mjr:selection-changed", { + selectedIds: newSelection, + selectedAssets: [asset], + activeId: newSelection[0] + }, { target: gridContainer, warnPrefix: "[GridContextMenu]" }); + } } catch {} // Update local set for menu logic selectedIds = new Set(newSelection); @@ -342,9 +388,15 @@ export function bindGridContextMenu({ // Visually update immediately (toggle class) to give feedback try { // Clear others - gridContainer.querySelectorAll('.mjr-asset-card.is-selected').forEach(el => el.classList.remove('is-selected')); + gridContainer.querySelectorAll('.mjr-asset-card.is-selected').forEach(el => { + el.classList.remove('is-selected'); + el.setAttribute?.('aria-selected', 'false'); + }); // Select target - if (card) card.classList.add('is-selected'); + if (card) { + card.classList.add('is-selected'); + card.setAttribute('aria-selected', 'true'); + } } catch {} } @@ -691,21 +743,27 @@ export function bindGridContextMenu({ // Delete each asset individually let successCount = 0; let errorCount = 0; + const deletedIds = []; for (const assetId of selectedIds) { const result = await deleteAsset(assetId); if (result?.ok) { successCount++; - // Remove card from DOM - const card = document.querySelector(`[data-mjr-asset-id="${safeEscapeSelector(assetId)}"]`); - if (card) card.remove(); + deletedIds.push(String(assetId)); } else { errorCount++; } } - // Clear selection - clearSelection(gridContainer); + if (deletedIds.length) { + const removal = removeAssetsFromGrid(gridContainer, deletedIds); + try { + if (panelState && Array.isArray(removal?.selectedIds)) { + panelState.selectedAssetIds = removal.selectedIds; + panelState.activeAssetId = removal.selectedAssetIds[0] || ""; + } + } catch {} + } if (errorCount === 0) { comfyToast(`${successCount} files deleted successfully!`, "success"); @@ -724,10 +782,12 @@ export function bindGridContextMenu({ try { const result = await deleteAsset(asset.id); if (result?.ok) { - // Remove the card from DOM + const removal = removeAssetsFromGrid(gridContainer, [String(asset.id)]); try { - const cardToRemove = document.querySelector(`[data-mjr-asset-id="${safeEscapeSelector(asset.id)}"]`); - cardToRemove?.remove?.(); + if (panelState && Array.isArray(removal?.selectedIds)) { + panelState.selectedAssetIds = removal.selectedIds; + panelState.activeAssetId = removal.selectedAssetIds[0] || ""; + } } catch {} comfyToast("File deleted successfully!", "success"); } else { diff --git a/js/features/dnd/out/DragOut.js b/js/features/dnd/out/DragOut.js index 940c888b..661c937c 100644 --- a/js/features/dnd/out/DragOut.js +++ b/js/features/dnd/out/DragOut.js @@ -57,7 +57,30 @@ const _getSelectedAssets = (containerEl, draggedCard) => { const out = []; if (!containerEl || !draggedCard) return out; - // Prefer DOM selection state (fast, always available). + let selectedIds = []; + try { + const raw = containerEl.dataset?.mjrSelectedAssetIds; + if (raw) { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) selectedIds = parsed.map(String).filter(Boolean); + } + } catch {} + + // If dataset is available, use full asset list (VirtualGrid-safe). + if (selectedIds.length) { + try { + const draggedId = draggedCard?.dataset?.mjrAssetId ? String(draggedCard.dataset.mjrAssetId) : ""; + if (!draggedId || !selectedIds.includes(draggedId)) return out; + const allAssets = typeof containerEl?._mjrGetAssets === "function" ? containerEl._mjrGetAssets() : []; + for (const a of allAssets || []) { + const id = String(a?.id || ""); + if (id && selectedIds.includes(id)) out.push(a); + } + if (out.length) return out; + } catch {} + } + + // Fallback: DOM selection state (visible cards only). let selectedCards = []; try { selectedCards = Array.from(containerEl.querySelectorAll(".mjr-asset-card.is-selected")) || []; diff --git a/js/features/grid/GridKeyboard.js b/js/features/grid/GridKeyboard.js index a55026f8..84fcf9a2 100644 --- a/js/features/grid/GridKeyboard.js +++ b/js/features/grid/GridKeyboard.js @@ -266,9 +266,17 @@ export function installGridKeyboard({ viewer.open(selected, 0); } else { // Open all assets from grid, starting at active - const allAssets = Array.from(gridContainer.querySelectorAll('.mjr-asset-card')) - .map(card => card._mjrAsset) - .filter(Boolean); + let allAssets = []; + if (Array.isArray(state?.assets) && state.assets.length) { + allAssets = state.assets; + } else if (typeof gridContainer?._mjrGetAssets === "function") { + const list = gridContainer._mjrGetAssets(); + if (Array.isArray(list) && list.length) allAssets = list; + } else { + allAssets = Array.from(gridContainer.querySelectorAll('.mjr-asset-card')) + .map(card => card._mjrAsset) + .filter(Boolean); + } const idx = allAssets.findIndex(a => String(a?.id) === String(asset?.id)); viewer.open(allAssets, Math.max(0, idx)); } @@ -428,6 +436,7 @@ export function installGridKeyboard({ const toDelete = selected.length > 0 ? selected : [asset]; let successCount = 0; let errorCount = 0; + const deletedIds = []; for (const a of toDelete) { if (!a?.id) continue; @@ -435,9 +444,7 @@ export function installGridKeyboard({ const result = await deleteAsset(a.id); if (result?.ok) { successCount++; - // Remove card from DOM - const card = gridContainer.querySelector(`[data-mjr-asset-id="${a.id}"]`); - card?.remove?.(); + deletedIds.push(String(a.id)); } else { errorCount++; } @@ -446,11 +453,31 @@ export function installGridKeyboard({ } } + if (deletedIds.length) { + try { + const removeFn = gridContainer?._mjrRemoveAssets; + if (typeof removeFn === "function") { + removeFn(deletedIds); + } else { + for (const id of deletedIds) { + const card = gridContainer.querySelector(`[data-mjr-asset-id="${id}"]`); + card?.remove?.(); + } + } + } catch {} + } + if (errorCount === 0) { comfyToast(`${successCount} file${successCount !== 1 ? 's' : ''} deleted`, "success"); } else { comfyToast(`${successCount} deleted, ${errorCount} failed`, "warning"); } + try { + if (typeof gridContainer?._mjrSetSelection === "function") { + const remaining = selected.filter(a => !deletedIds.includes(String(a?.id || ""))).map(a => String(a?.id || "")).filter(Boolean); + gridContainer._mjrSetSelection(remaining, remaining[0] || ""); + } + } catch {} onSelectionChanged(); return; } @@ -459,14 +486,34 @@ export function installGridKeyboard({ // Select All (Ctrl+A) if (matchesShortcut(e, GRID_SHORTCUTS.SELECT_ALL)) { consume(); - const cards = gridContainer.querySelectorAll('.mjr-asset-card'); - const ids = []; - cards.forEach(card => { - card.classList.add('is-selected'); - card.setAttribute('aria-selected', 'true'); - if (card._mjrAsset?.id) ids.push(String(card._mjrAsset.id)); - }); - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(ids); + let ids = []; + try { + if (Array.isArray(state?.assets) && state.assets.length) { + ids = state.assets.map(a => String(a?.id || "")).filter(Boolean); + } else if (typeof gridContainer?._mjrGetAssets === "function") { + const list = gridContainer._mjrGetAssets(); + if (Array.isArray(list)) ids = list.map(a => String(a?.id || "")).filter(Boolean); + } + } catch {} + if (!ids.length) { + try { + const cards = gridContainer.querySelectorAll('.mjr-asset-card'); + ids = Array.from(cards).map(card => String(card?._mjrAsset?.id || "")).filter(Boolean); + } catch {} + } + try { + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection(ids, ids[0] || ""); + } else { + const cards = gridContainer.querySelectorAll('.mjr-asset-card'); + cards.forEach(card => { + card.classList.add('is-selected'); + card.setAttribute('aria-selected', 'true'); + }); + gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(ids); + gridContainer.dataset.mjrSelectedAssetId = String(ids[0] || ""); + } + } catch {} onSelectionChanged(); return; } @@ -474,12 +521,19 @@ export function installGridKeyboard({ // Deselect All (Ctrl+D) if (matchesShortcut(e, GRID_SHORTCUTS.DESELECT_ALL)) { consume(); - const cards = gridContainer.querySelectorAll('.mjr-asset-card.is-selected'); - cards.forEach(card => { - card.classList.remove('is-selected'); - card.setAttribute('aria-selected', 'false'); - }); - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify([]); + try { + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection([], ""); + } else { + const cards = gridContainer.querySelectorAll('.mjr-asset-card.is-selected'); + cards.forEach(card => { + card.classList.remove('is-selected'); + card.setAttribute('aria-selected', 'false'); + }); + delete gridContainer.dataset.mjrSelectedAssetIds; + delete gridContainer.dataset.mjrSelectedAssetId; + } + } catch {} onSelectionChanged(); return; } diff --git a/js/features/grid/GridView.js b/js/features/grid/GridView.js index be355153..58072809 100644 --- a/js/features/grid/GridView.js +++ b/js/features/grid/GridView.js @@ -375,6 +375,22 @@ export function createGridContainer() { const container = document.createElement("div"); container.id = "mjr-assets-grid"; container.classList.add("mjr-grid"); + + // Provide a stable accessor for full asset lists (VirtualGrid-safe). + try { + container._mjrGetAssets = () => { + try { + const state = GRID_STATE.get(container); + return Array.isArray(state?.assets) ? state.assets : []; + } catch { + return []; + } + }; + } catch {} + try { + container._mjrSetSelection = (ids, activeId = "") => setSelectionIds(container, ids, { activeId }); + container._mjrSyncSelection = (ids) => syncSelectionClasses(container, ids); + } catch {} // Initial class application _updateGridSettingsClasses(container); @@ -707,8 +723,26 @@ function ensureVirtualGrid(gridContainer, state) { return card; }, onItemRendered: (asset, card) => { + // Ensure selection classes are clean before applying any selection logic. + try { + card.classList.remove("is-selected"); + card.classList.remove("is-active"); + card.setAttribute("aria-selected", "false"); + } catch {} + // Re-apply selection if this asset is selected. + try { + const selectedIds = getSelectedIdSet(gridContainer); + if (asset?.id != null && selectedIds.has(String(asset.id))) { + card.classList.add("is-selected"); + card.setAttribute("aria-selected", "true"); + } + } catch {} + // Hydrate ratings enqueueRatingTagsHydration(gridContainer, card, asset); + try { + state.virtualGrid?.scheduleRemeasure?.(); + } catch {} // Track rendered card for live updates (collisions) const fnKey = _getFilenameKey(asset?.filename); @@ -725,6 +759,15 @@ function ensureVirtualGrid(gridContainer, state) { if (card) { card._mjrAsset = asset; _updateCardRatingTagsBadges(card, asset.rating, asset.tags); + try { + state.virtualGrid?.scheduleRemeasure?.(); + } catch {} + // Reset selection classes on reuse to avoid ghost selection + try { + card.classList.remove("is-selected"); + card.classList.remove("is-active"); + card.setAttribute("aria-selected", "false"); + } catch {} const selectedIds = getSelectedIdSet(gridContainer); if (selectedIds && asset.id != null) { @@ -760,6 +803,11 @@ function ensureVirtualGrid(gridContainer, state) { try { cleanupCardMediaHandlers(card); } catch {} + try { + card.classList.remove("is-selected"); + card.classList.remove("is-active"); + card.setAttribute("aria-selected", "false"); + } catch {} } }); @@ -813,6 +861,60 @@ function getSelectedIdSet(gridContainer) { return selected; } +function _setSelectedIdsDataset(gridContainer, selectedSet, activeId = "") { + const list = Array.from(selectedSet || []); + const active = activeId ? String(activeId) : (list[0] ? String(list[0]) : ""); + try { + if (list.length) { + gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(list); + if (active) gridContainer.dataset.mjrSelectedAssetId = String(active); + else delete gridContainer.dataset.mjrSelectedAssetId; + } else { + delete gridContainer.dataset.mjrSelectedAssetIds; + delete gridContainer.dataset.mjrSelectedAssetId; + } + } catch {} + return list; +} + +export function syncSelectionClasses(gridContainer, selectedIds) { + if (!gridContainer) return; + const set = selectedIds instanceof Set ? selectedIds : new Set(Array.from(selectedIds || []).map(String)); + try { + for (const card of gridContainer.querySelectorAll(".mjr-asset-card")) { + const id = card?.dataset?.mjrAssetId; + if (id && set.has(String(id))) { + card.classList.add("is-selected"); + card.setAttribute("aria-selected", "true"); + } else { + card.classList.remove("is-selected"); + card.setAttribute("aria-selected", "false"); + } + } + } catch {} +} + +export function setSelectionIds(gridContainer, selectedIds, { activeId = "" } = {}) { + if (!gridContainer) return []; + const set = new Set(Array.from(selectedIds || []).map(String).filter(Boolean)); + const list = _setSelectedIdsDataset(gridContainer, set, activeId); + syncSelectionClasses(gridContainer, set); + try { + gridContainer.dispatchEvent?.( + new CustomEvent("mjr:selection-changed", { detail: { selectedIds: list, activeId: activeId || list[0] || "" } }) + ); + } catch {} + return list; +} + +function _safeEscapeId(value) { + try { + return CSS?.escape ? CSS.escape(String(value)) : String(value).replace(/["\\]/g, "\\$&"); + } catch { + return String(value).replace(/["\\]/g, "\\$&"); + } +} + function _getFilenameKey(filename) { try { return String(filename || "").trim().toLowerCase(); @@ -1607,6 +1709,110 @@ export async function loadAssetsFromList(gridContainer, assets, options = {}) { } } +/** + * Remove assets from grid state + UI (used after delete). + * Keeps selection dataset consistent and avoids UI re-appearance on scroll. + * @param {HTMLElement} gridContainer + * @param {Array} assetIds + * @param {Object} [options] + * @param {boolean} [options.updateSelection=true] + * @returns {{ok: boolean, removed: number, selectedIds: string[]}} + */ +export function removeAssetsFromGrid(gridContainer, assetIds, { updateSelection = true } = {}) { + if (!gridContainer) { + return { ok: false, removed: 0, selectedIds: [] }; + } + const ids = new Set(); + const list = Array.isArray(assetIds) ? assetIds : [assetIds]; + for (const raw of list) { + if (raw == null) continue; + if (typeof raw === "object" && raw?.id != null) { + ids.add(String(raw.id)); + } else { + ids.add(String(raw)); + } + } + if (!ids.size) { + return { ok: false, removed: 0, selectedIds: Array.from(getSelectedIdSet(gridContainer)) }; + } + + let selectedIds = getSelectedIdSet(gridContainer); + if (updateSelection) { + for (const id of ids) selectedIds.delete(String(id)); + _setSelectedIdsDataset(gridContainer, selectedIds); + } + + // Best-effort selection class sync for visible cards. + try { + for (const card of gridContainer.querySelectorAll(".mjr-asset-card.is-selected")) { + const id = card?.dataset?.mjrAssetId; + if (!id || !selectedIds.has(String(id))) { + card.classList.remove("is-selected"); + card.setAttribute("aria-selected", "false"); + } + } + } catch {} + + const state = GRID_STATE.get(gridContainer); + let removedCount = 0; + if (state?.assets && state.assets.length) { + const next = []; + for (const asset of state.assets) { + const id = asset?.id != null ? String(asset.id) : ""; + if (id && ids.has(id)) { + removedCount += 1; + } else { + next.push(asset); + } + } + if (removedCount) { + state.assets = next; + try { + if (Number.isFinite(state.total)) state.total = Math.max(0, Number(state.total) - removedCount); + } catch {} + try { + if (Number.isFinite(state.offset)) state.offset = Math.max(0, Number(state.offset) - removedCount); + } catch {} + if (state.virtualGrid) { + state.virtualGrid.setItems(state.assets); + } else { + for (const id of ids) { + try { + const card = gridContainer.querySelector(`[data-mjr-asset-id="${_safeEscapeId(id)}"]`); + card?.remove?.(); + } catch {} + } + } + } + } else { + for (const id of ids) { + try { + const card = gridContainer.querySelector(`[data-mjr-asset-id="${_safeEscapeId(id)}"]`); + if (card) { + card.remove?.(); + removedCount += 1; + } + } catch {} + } + } + + const selectedList = Array.from(selectedIds); + try { + gridContainer?.dispatchEvent?.(new CustomEvent("mjr:selection-changed", { detail: { selectedIds: selectedList } })); + } catch {} + try { + const count = Number(state?.assets?.length ?? null); + const total = Number(state?.total ?? null); + if (Number.isFinite(count) || Number.isFinite(total)) { + gridContainer?.dispatchEvent?.( + new CustomEvent("mjr:grid-stats", { detail: { count, total } }) + ); + } + } catch {} + + return { ok: true, removed: removedCount, selectedIds: selectedList }; +} + export function disposeGrid(gridContainer) { const state = GRID_STATE.get(gridContainer); if (!state) return; @@ -1778,11 +1984,18 @@ function _flushUpsertBatch(gridContainer) { state.seenKeys.add(key); + // If insertion is near the top, mark re-sort to avoid large index shifts. + const shouldResort = insertPos <= Math.min(10, state.assets.length); if (insertPos === -1) { state.assets.push(asset); } else { state.assets.splice(insertPos, 0, asset); } + if (shouldResort) { + try { + state._mjrResortPending = true; + } catch {} + } modified = true; } } @@ -1790,6 +2003,20 @@ function _flushUpsertBatch(gridContainer) { // Single setItems call for all changes if (modified && vg) { + // If many top inserts occurred, full re-sort to stabilize indices. + if (state._mjrResortPending) { + try { + const sortKey = gridContainer.dataset.mjrSort || "mtime_desc"; + state.assets.sort((a, b) => { + const av = getSortValue(a, sortKey); + const bv = getSortValue(b, sortKey); + if (av < bv) return -1; + if (av > bv) return 1; + return 0; + }); + } catch {} + try { state._mjrResortPending = false; } catch {} + } vg.setItems(state.assets); } } catch (err) { @@ -1973,7 +2200,7 @@ export function refreshGrid(gridContainer) { // [ISSUE 1] Real-time Refresh Implementation // Listens for the event dispatched by entry.js (relayed from backend) try { - window.addEventListener("mjr:scan-complete", (e) => { + const onScanComplete = (e) => { const detail = e.detail; // Find all active grids @@ -2003,7 +2230,11 @@ try { console.warn("[Majoor] Auto-refresh error:", err); } } - }); + }; + + // Canonical event name: mjr-scan-complete (keep legacy alias for safety). + window.addEventListener("mjr-scan-complete", onScanComplete); + window.addEventListener("mjr:scan-complete", onScanComplete); } catch (e) { console.warn("Failed to register global scan listener:", e); } diff --git a/js/features/grid/VirtualGrid.js b/js/features/grid/VirtualGrid.js index 9683aea0..28d1749c 100644 --- a/js/features/grid/VirtualGrid.js +++ b/js/features/grid/VirtualGrid.js @@ -107,8 +107,9 @@ export class VirtualGrid { // Style the container for absolute positioning this.container.style.position = "relative"; this.container.style.display = "block"; // Override 'grid' - // Force no horizontal scrollbar - this.container.style.overflowX = "hidden"; + // Use overflow: visible so IntersectionObserver can properly detect sentinel + // Parent wrapper (gridWrapper) handles scrolling and clipping with overflow: auto + this.container.style.overflow = "visible"; this.container.style.width = "100%"; } @@ -306,6 +307,22 @@ export class VirtualGrid { } catch {} } + /** + * Best-effort re-measure after async content (badges/metadata) is applied. + * This avoids layout drift when card height changes after initial render. + */ + scheduleRemeasure() { + if (this.measured) return; + try { + requestAnimationFrame(() => { + try { this.updateMetrics(); } catch {} + }); + setTimeout(() => { + try { this.updateMetrics(); } catch {} + }, 60); + } catch {} + } + /** * Core render loop. Calculates visible range and creates/removes DOM elements. */ diff --git a/js/features/panel/AssetsManagerPanel.js b/js/features/panel/AssetsManagerPanel.js index 83f7383f..577bb93d 100644 --- a/js/features/panel/AssetsManagerPanel.js +++ b/js/features/panel/AssetsManagerPanel.js @@ -3,7 +3,7 @@ import { comfyConfirm, comfyPrompt } from "../../app/dialogs.js"; import { comfyToast } from "../../app/toast.js"; import { createStatusIndicator, setupStatusPolling, triggerScan } from "../status/StatusDot.js"; import { createGridContainer, loadAssets, loadAssetsFromList, disposeGrid, refreshGrid } from "../grid/GridView.js"; -import { get, post, getCollectionAssets } from "../../api/client.js"; +import { get, post, getCollectionAssets, toggleWatcher, setWatcherScope } from "../../api/client.js"; import { ENDPOINTS } from "../../api/endpoints.js"; import { createSidebar, showAssetInSidebar, closeSidebar } from "../../components/SidebarView.js"; import { createRatingBadge, createTagsBadge } from "../../components/Badges.js"; @@ -221,6 +221,21 @@ export async function renderAssetsManager(container, { useComfyThemeUI = true } } }; + const applyWatcherForScope = async (nextScope) => { + const scope = String(nextScope || state.scope || "output").toLowerCase(); + const settings = loadMajoorSettings(); + const wantsEnabled = !!settings?.watcher?.enabled; + const shouldEnable = wantsEnabled && scope === "output"; + try { + await toggleWatcher(shouldEnable); + } catch {} + if (shouldEnable) { + try { + await setWatcherScope({ scope: "output" }); + } catch {} + } + }; + // Initial position (no reload needed). try { _sidebarPosition = ""; // force apply @@ -239,6 +254,9 @@ export async function renderAssetsManager(container, { useComfyThemeUI = true } // Use key from event to optimize? For now, refresh is cheap (VirtualGrid). refreshGrid(gridWrapper); } catch {} + try { + applyWatcherForScope(state.scope); + } catch {} }; try { window.addEventListener?.("mjr-settings-changed", onSettingsChanged); @@ -304,6 +322,11 @@ export async function renderAssetsManager(container, { useComfyThemeUI = true } reloadGrid: gridController.reloadGrid, onChanged: () => { notifyContextChanged(); + }, + onScopeChanged: async () => { + try { + await applyWatcherForScope(state.scope); + } catch {} } }); @@ -422,7 +445,17 @@ export async function renderAssetsManager(container, { useComfyThemeUI = true } try { notifyContextChanged(); const onStats = () => notifyContextChanged(); + const onSelectionChanged = (e) => { + try { + const detail = e?.detail || {}; + const ids = Array.isArray(detail.selectedIds) ? detail.selectedIds.map(String).filter(Boolean) : []; + state.selectedAssetIds = ids; + state.activeAssetId = String(detail.activeId || ids[0] || ""); + } catch {} + notifyContextChanged(); + }; gridContainer.addEventListener("mjr:grid-stats", onStats); + gridContainer.addEventListener("mjr:selection-changed", onSelectionChanged); document.addEventListener?.("mjr-settings-changed", onStats); // Observe DOM changes for card count / selection class changes. @@ -441,6 +474,9 @@ export async function renderAssetsManager(container, { useComfyThemeUI = true } try { gridContainer.removeEventListener("mjr:grid-stats", onStats); } catch {} + try { + gridContainer.removeEventListener("mjr:selection-changed", onSelectionChanged); + } catch {} try { document.removeEventListener?.("mjr-settings-changed", onStats); } catch {} @@ -671,13 +707,17 @@ export async function renderAssetsManager(container, { useComfyThemeUI = true } selected.forEach(id => { const card = gridContainer.querySelector(`.mjr-asset-card[data-mjr-asset-id="${CSS?.escape ? CSS.escape(id) : id}"]`); - if (card) card.classList.add("is-selected"); + if (card) { + card.classList.add("is-selected"); + card.setAttribute?.("aria-selected", "true"); + } }); if (activeId) { const activeCard = gridContainer.querySelector(`.mjr-asset-card[data-mjr-asset-id="${CSS?.escape ? CSS.escape(activeId) : activeId}"]`); if (activeCard) { activeCard.classList.add("is-active"); + activeCard.setAttribute?.("aria-selected", "true"); // Scroll selection into view if not visible after scroll restore if (state.scrollTop === 0) { activeCard.scrollIntoView?.({ block: "nearest", behavior: "instant" }); diff --git a/js/features/panel/controllers/contextController.js b/js/features/panel/controllers/contextController.js index b343719f..44926806 100644 --- a/js/features/panel/controllers/contextController.js +++ b/js/features/panel/controllers/contextController.js @@ -51,21 +51,39 @@ export function createContextController({ } catch {} }, clearScope: async () => { + let didSetScope = false; try { - state.scope = "output"; state.customRootId = ""; - scopeController?.setActiveTabStyles?.(); + state.subfolder = ""; + delete gridContainer?.dataset?.mjrSubfolder; + if (typeof scopeController?.setScope === "function") { + await scopeController.setScope("output"); + didSetScope = true; + } else { + state.scope = "output"; + scopeController?.setActiveTabStyles?.(); + } } catch {} try { - await reloadGrid?.(); + if (!didSetScope) await reloadGrid?.(); } catch {} }, clearCustomRoot: async () => { + let didSetScope = false; try { state.customRootId = ""; + if (String(state.scope || "").toLowerCase() === "custom") { + if (typeof scopeController?.setScope === "function") { + await scopeController.setScope("output"); + didSetScope = true; + } else { + state.scope = "output"; + scopeController?.setActiveTabStyles?.(); + } + } } catch {} try { - await reloadGrid?.(); + if (!didSetScope) await reloadGrid?.(); } catch {} }, clearFolder: async () => { @@ -134,6 +152,7 @@ export function createContextController({ } catch {} }, clearAll: async () => { + let didSetScope = false; try { _safeSetValue(searchInputEl, ""); } catch {} @@ -147,8 +166,15 @@ export function createContextController({ state.dateExactFilter = ""; state.sort = "mtime_desc"; state.customRootId = ""; - state.scope = "output"; - scopeController?.setActiveTabStyles?.(); + state.subfolder = ""; + delete gridContainer?.dataset?.mjrSubfolder; + if (typeof scopeController?.setScope === "function") { + await scopeController.setScope("output"); + didSetScope = true; + } else { + state.scope = "output"; + scopeController?.setActiveTabStyles?.(); + } } catch {} try { _safeSetValue(kindSelect, ""); @@ -162,7 +188,7 @@ export function createContextController({ sortController?.renderSortMenu?.(); } catch {} try { - await reloadGrid?.(); + if (!didSetScope) await reloadGrid?.(); } catch {} } }; diff --git a/js/features/panel/controllers/customRootsController.js b/js/features/panel/controllers/customRootsController.js index 078e61c4..dfa2ce6a 100644 --- a/js/features/panel/controllers/customRootsController.js +++ b/js/features/panel/controllers/customRootsController.js @@ -10,7 +10,8 @@ export function createCustomRootsController({ get, post, ENDPOINTS, - reloadGrid + reloadGrid, + onRootChanged = null }) { const isSensitivePath = (input) => { try { @@ -98,6 +99,9 @@ export function createCustomRootsController({ if (!customSelect.disabled) { state.customRootId = customSelect.value || ""; customRemoveBtn.disabled = !state.customRootId; + try { + await onRootChanged?.(state); + } catch {} await reloadGrid(); } }); diff --git a/js/features/panel/controllers/scopeController.js b/js/features/panel/controllers/scopeController.js index f47e168c..59722750 100644 --- a/js/features/panel/controllers/scopeController.js +++ b/js/features/panel/controllers/scopeController.js @@ -1,4 +1,4 @@ -export function createScopeController({ state, tabButtons, customMenuBtn, customPopover, popovers, refreshCustomRoots, reloadGrid, onChanged = null }) { +export function createScopeController({ state, tabButtons, customMenuBtn, customPopover, popovers, refreshCustomRoots, reloadGrid, onChanged = null, onScopeChanged = null }) { const setActiveTabStyles = () => { const buttons = Object.values(tabButtons || {}); buttons.forEach((b) => { @@ -34,6 +34,9 @@ export function createScopeController({ state, tabButtons, customMenuBtn, custom try { onChanged?.(); } catch {} + try { + await onScopeChanged?.(state); + } catch {} await reloadGrid(); }; diff --git a/js/features/panel/controllers/sidebarController.js b/js/features/panel/controllers/sidebarController.js index 3017da4c..1a9cb122 100644 --- a/js/features/panel/controllers/sidebarController.js +++ b/js/features/panel/controllers/sidebarController.js @@ -2,6 +2,7 @@ import { createWorkflowDot } from "../../../components/Badges.js"; import { post, getAssetMetadata } from "../../../api/client.js"; import { ENDPOINTS } from "../../../api/endpoints.js"; import { pickRootId } from "../../../utils/ids.js"; +import { safeClosest } from "../../../utils/dom.js"; const RESCAN_FLAG = "_mjrRescanning"; const RESCAN_TTL_MS = 1500; @@ -121,10 +122,31 @@ function ensureSelectionVisible(gridContainer) { ) : null); if (!selected) return; + const root = gridContainer.parentElement; + if (root && typeof root.getBoundingClientRect === "function" && typeof selected.getBoundingClientRect === "function") { + const rootRect = root.getBoundingClientRect(); + const selRect = selected.getBoundingClientRect(); + const fullyVisible = selRect.top >= rootRect.top && selRect.bottom <= rootRect.bottom; + if (fullyVisible) return; + } selected.scrollIntoView?.({ block: "nearest", inline: "nearest" }); } catch {} } +function scheduleEnsureSelectionVisible(gridContainer) { + try { + requestAnimationFrame(() => { + try { ensureSelectionVisible(gridContainer); } catch {} + // Second pass after layout settles (sidebar open/close). + setTimeout(() => { + try { ensureSelectionVisible(gridContainer); } catch {} + }, 50); + }); + } catch { + try { ensureSelectionVisible(gridContainer); } catch {} + } +} + function setCardSelected(card, selected) { if (!card) return; if (selected) { @@ -139,6 +161,10 @@ function setCardSelected(card, selected) { function clearAllSelected(gridContainer) { if (!gridContainer) return; try { + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection([], ""); + return; + } for (const el of gridContainer.querySelectorAll(".mjr-asset-card.is-selected")) { setCardSelected(el, false); } @@ -159,10 +185,50 @@ function updateSelectedIdsDataset(gridContainer) { const ids = Array.from(gridContainer.querySelectorAll(".mjr-asset-card.is-selected")) .map((c) => c?.dataset?.mjrAssetId) .filter(Boolean); - gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(ids); + if (ids.length) { + gridContainer.dataset.mjrSelectedAssetIds = JSON.stringify(ids); + if (!gridContainer.dataset.mjrSelectedAssetId && ids[0]) { + gridContainer.dataset.mjrSelectedAssetId = String(ids[0]); + } + } else { + delete gridContainer.dataset.mjrSelectedAssetIds; + delete gridContainer.dataset.mjrSelectedAssetId; + } } catch {} } +function applySelection(gridContainer, ids, activeId = "") { + if (!gridContainer) return []; + const list = Array.from(ids || []).map(String).filter(Boolean); + try { + if (typeof gridContainer?._mjrSetSelection === "function") { + gridContainer._mjrSetSelection(list, activeId || list[0] || ""); + return list; + } + } catch {} + try { + gridContainer.dataset.mjrSelectedAssetIds = list.length ? JSON.stringify(list) : ""; + if (list.length) gridContainer.dataset.mjrSelectedAssetId = String(activeId || list[0] || ""); + else { + delete gridContainer.dataset.mjrSelectedAssetIds; + delete gridContainer.dataset.mjrSelectedAssetId; + } + } catch {} + try { + for (const card of gridContainer.querySelectorAll(".mjr-asset-card")) { + const id = card?.dataset?.mjrAssetId; + const selected = id && list.includes(String(id)); + setCardSelected(card, !!selected); + } + } catch {} + try { + gridContainer.dispatchEvent?.( + new CustomEvent("mjr:selection-changed", { detail: { selectedIds: list, activeId: activeId || list[0] || "" } }) + ); + } catch {} + return list; +} + function syncSelectionState({ gridContainer, state, activeId } = {}) { if (!state || !gridContainer) return; try { @@ -270,11 +336,11 @@ export function bindSidebarOpen({ const onClick = async (e) => { if (e.defaultPrevented) return; - const card = e.target.closest(".mjr-asset-card"); + const card = safeClosest(e.target, ".mjr-asset-card"); if (!card) return; // Click on workflow dot triggers a forced re-index of this one file (no directory scan). - const dot = e.target.closest(".mjr-workflow-dot"); + const dot = safeClosest(e.target, ".mjr-workflow-dot"); if (dot && card.contains(dot)) { const asset = card._mjrAsset; if (!asset) return; @@ -283,10 +349,9 @@ export function bindSidebarOpen({ e.stopPropagation(); // Ensure this card is selected, but do not open the sidebar from dot clicks. - clearAllSelected(gridContainer); - setSelectedCard(gridContainer, card); - updateSelectedIdsDataset(gridContainer); - syncSelectionState({ gridContainer, state, activeId: card?.dataset?.mjrAssetId }); + const id = card?.dataset?.mjrAssetId || ""; + applySelection(gridContainer, id ? [id] : [], id); + syncSelectionState({ gridContainer, state, activeId: id }); const updated = await rescanSingleAsset({ card, @@ -306,7 +371,7 @@ export function bindSidebarOpen({ return; } - const interactive = e.target.closest("a, button, input, textarea, select, label"); + const interactive = safeClosest(e.target, "a, button, input, textarea, select, label"); if (interactive && card.contains(interactive)) return; const asset = card._mjrAsset; @@ -317,10 +382,16 @@ export function bindSidebarOpen({ const isMulti = !!(e.ctrlKey || e.metaKey || e.shiftKey); if (!isMulti) { - clearAllSelected(gridContainer); - setSelectedCard(gridContainer, card); - updateSelectedIdsDataset(gridContainer); - syncSelectionState({ gridContainer, state, activeId: card?.dataset?.mjrAssetId }); + const alreadySelected = card.classList.contains("is-selected"); + if (alreadySelected) { + applySelection(gridContainer, [], ""); + syncSelectionState({ gridContainer, state, activeId: "" }); + return; + } + + const id = card?.dataset?.mjrAssetId; + applySelection(gridContainer, id ? [id] : [], id || ""); + syncSelectionState({ gridContainer, state, activeId: id || "" }); // When the sidebar is already open, clicking a different card navigates details. try { @@ -332,19 +403,28 @@ export function bindSidebarOpen({ const cards = getAllCards(gridContainer); const idx = cards.indexOf(card); - if (e.shiftKey && idx >= 0) { - const last = Number(gridContainer._mjrLastSelectedIndex); - const anchor = Number.isFinite(last) ? last : idx; - clearAllSelected(gridContainer); - const start = Math.min(anchor, idx); - const end = Math.max(anchor, idx); - for (let i = start; i <= end; i++) { - setCardSelected(cards[i], true); + if (e.shiftKey) { + let allAssets = []; + try { + if (typeof gridContainer?._mjrGetAssets === "function") { + allAssets = gridContainer._mjrGetAssets() || []; + } + } catch {} + if (Array.isArray(allAssets) && allAssets.length) { + const id = card?.dataset?.mjrAssetId || ""; + const targetIndex = allAssets.findIndex(a => String(a?.id || "") === String(id)); + const last = Number(gridContainer._mjrLastSelectedIndex); + const anchor = Number.isFinite(last) ? last : targetIndex; + if (targetIndex >= 0 && anchor >= 0) { + const start = Math.min(anchor, targetIndex); + const end = Math.max(anchor, targetIndex); + const rangeIds = allAssets.slice(start, end + 1).map(a => String(a?.id || "")).filter(Boolean); + applySelection(gridContainer, rangeIds, id || rangeIds[0] || ""); + gridContainer._mjrLastSelectedIndex = targetIndex; + syncSelectionState({ gridContainer, state, activeId: id || "" }); + return; + } } - gridContainer._mjrLastSelectedIndex = idx; - updateSelectedIdsDataset(gridContainer); - syncSelectionState({ gridContainer, state, activeId: card?.dataset?.mjrAssetId }); - return; } if (idx >= 0) { @@ -398,7 +478,7 @@ export function bindSidebarOpen({ await showAssetInSidebar(sidebar, asset, (updatedAsset) => applyAssetUpdateToCard(card, asset, updatedAsset)); // Persist sidebar open state try { state.sidebarOpen = true; } catch {} - ensureSelectionVisible(gridContainer); + scheduleEnsureSelectionVisible(gridContainer); card?.focus?.({ preventScroll: true }); } catch {} }; @@ -413,7 +493,7 @@ export function bindSidebarOpen({ e.target?.isContentEditable || e.target?.closest?.("input, textarea, select, [contenteditable='true']"); if (isTypingTarget) return; - const card = e.target?.closest?.(".mjr-asset-card"); + const card = safeClosest(e.target, ".mjr-asset-card"); if (!card) return; const asset = card._mjrAsset; if (!asset) return; diff --git a/js/features/status/StatusDot.js b/js/features/status/StatusDot.js index 8de517d5..f007f480 100644 --- a/js/features/status/StatusDot.js +++ b/js/features/status/StatusDot.js @@ -2,7 +2,7 @@ * Status Dot Feature - Health indicator */ -import { get, getToolsStatus, post, resetIndex } from "../../api/client.js"; +import { get, getToolsStatus, post, resetIndex, getWatcherStatus } from "../../api/client.js"; import { ENDPOINTS } from "../../api/endpoints.js"; import { APP_CONFIG } from "../../app/config.js"; import { comfyToast } from "../../app/toast.js"; @@ -53,10 +53,42 @@ function setStatusWithHint(statusText, mainText, hintText) { statusText.appendChild(span); } +function formatWatcherScopeLabel(scope) { + const s = String(scope || "").toLowerCase(); + if (s === "all") return t("scope.allFull", "All (Inputs + Outputs)"); + if (s === "input" || s === "inputs") return t("scope.input", "Inputs"); + if (s === "custom") return t("scope.custom", "Custom"); + return t("scope.output", "Outputs"); +} + +function formatWatcherLine(watcher, desiredScope = "") { + if (!watcher || typeof watcher !== "object") { + if (desiredScope) { + return t("status.watcher.disabledScoped", "Watcher: disabled ({scope})", { scope: formatWatcherScopeLabel(desiredScope) }); + } + return t("status.watcher.disabled", "Watcher: disabled"); + } + const enabled = !!watcher.enabled; + const scope = watcher.scope ? String(watcher.scope) : ""; + if (!enabled) { + if (scope || desiredScope) { + const effective = desiredScope && scope && scope !== desiredScope ? desiredScope : (scope || desiredScope); + return t("status.watcher.disabledScoped", "Watcher: disabled ({scope})", { scope: formatWatcherScopeLabel(effective) }); + } + return t("status.watcher.disabled", "Watcher: disabled"); + } + if (scope) { + return t("status.watcher.enabledScoped", `Watcher: enabled (${scope})`, { scope: formatWatcherScopeLabel(scope) }); + } + return t("status.watcher.enabled", "Watcher: enabled"); +} + /** * Create status indicator section */ -export function createStatusIndicator() { +export function createStatusIndicator(options = {}) { + const getScanContext = + typeof options?.getScanContext === "function" ? options.getScanContext : null; const section = document.createElement("div"); section.style.cssText = "margin-bottom: 20px; padding: 12px; background: var(--bg-color, #1a1a1a); border-radius: 6px; border: 1px solid var(--border-color, #333); cursor: pointer; transition: all 0.2s;"; @@ -138,7 +170,7 @@ export function createStatusIndicator() { }; resetBtn.onclick = async (event) => { event.stopPropagation(); - + comfyToast(t("toast.resetTriggered"), "warning", 3000); const originalText = resetBtn.textContent; @@ -150,10 +182,16 @@ export function createStatusIndicator() { statusDot.style.background = "var(--mjr-status-warning, #FFA726)"; // Orange (working) try { + const ctx = getScanContext ? getScanContext() : {}; + const scope = String(ctx?.scope || "output").toLowerCase(); + const customRootId = ctx?.customRootId || ctx?.custom_root_id || ctx?.root_id || null; + const isAll = scope === "all"; + const res = await resetIndex({ - scope: "all", + scope, + customRootId, reindex: true, - hard_reset_db: true, + hard_reset_db: isAll, clear_scan_journal: true, clear_metadata_cache: true, clear_asset_metadata: true, @@ -216,6 +254,32 @@ export async function triggerScan(statusDot, statusText, capabilitiesSection = n const desiredScope = String(scanTarget?.scope || "output").toLowerCase(); const desiredCustomRootId = scanTarget?.customRootId || scanTarget?.custom_root_id || scanTarget?.root_id || null; + // Prevent overlapping scans (manual or polling-triggered). + try { + const now = Date.now(); + const inflight = globalThis?._mjrScanInFlight; + if (inflight && typeof inflight === "object") { + const ageMs = now - Number(inflight.at || 0); + if (ageMs >= 0 && ageMs < 10 * 60_000) { + setStatusWithHint( + statusText, + t("status.scanInProgress", "Scan already running..."), + t("status.scanInProgressHint", "Please wait for the current scan to finish") + ); + return; + } + } + globalThis._mjrScanInFlight = { at: now, scope: desiredScope, root_id: desiredCustomRootId || null }; + } catch {} + + const clearScanInFlight = () => { + try { + if (globalThis?._mjrScanInFlight) { + globalThis._mjrScanInFlight = null; + } + } catch {} + }; + // Incremental is always favored. // Shift+Click logic is handled by caller (if applicable), but here we default to true. const shouldIncremental = true; @@ -232,6 +296,7 @@ export async function triggerScan(statusDot, statusText, capabilitiesSection = n const configResult = await get(ENDPOINTS.CONFIG); if (!configResult.ok) { statusText.textContent = t("status.errorGetConfig"); + clearScanInFlight(); return; } roots = { output_directory: configResult.data.output_directory }; @@ -269,15 +334,21 @@ export async function triggerScan(statusDot, statusText, capabilitiesSection = n if (!desiredCustomRootId) { statusDot.style.background = "var(--mjr-status-error, #f44336)"; statusText.textContent = t("status.selectCustomFolder"); + clearScanInFlight(); return; } payload.custom_root_id = desiredCustomRootId; } - // Trigger scan - const scanResult = await post(ENDPOINTS.SCAN, payload); + let scanResult = null; + try { + // Trigger scan + scanResult = await post(ENDPOINTS.SCAN, payload); + } finally { + clearScanInFlight(); + } - if (scanResult.ok) { + if (scanResult?.ok) { const stats = scanResult.data; statusDot.style.background = "var(--mjr-status-success, #4CAF50)"; // Green setStatusWithHint( @@ -292,7 +363,7 @@ export async function triggerScan(statusDot, statusText, capabilitiesSection = n }, 2000); } else { statusDot.style.background = "var(--mjr-status-error, #f44336)"; // Red - setStatusLines(statusText, [t("toast.scanFailed") + `: ${scanResult.error || "Unknown error"}`]); + setStatusLines(statusText, [t("toast.scanFailed") + `: ${scanResult?.error || "Unknown error"}`]); // Restore status after 3 seconds setTimeout(() => { @@ -446,6 +517,20 @@ export async function updateStatus(statusDot, statusText, capabilitiesSection = const lastScanText = lastScanRaw ? new Date(lastScanRaw).toLocaleString() : "N/A"; const toolAvailability = counters.tool_availability || {}; const toolPaths = counters.tool_paths || {}; + let watcherInfo = counters.watcher; + try { + if (!watcherInfo || typeof watcherInfo.enabled !== "boolean") { + const wres = await getWatcherStatus(); + if (wres?.ok && wres.data) { + watcherInfo = { + enabled: !!wres.data.enabled, + scope: (counters?.watcher?.scope || desiredScope), + custom_root_id: desiredCustomRootId || null, + }; + } + } + } catch {} + const watcherLine = formatWatcherLine(watcherInfo, desiredScope); renderCapabilities(capabilitiesSection, toolAvailability, toolPaths); renderToolsStatusLine(capabilitiesSection, toolStatusData, toolAvailability); @@ -465,7 +550,7 @@ export async function updateStatus(statusDot, statusText, capabilitiesSection = setStatusWithHint( statusText, t("status.noAssets", `No assets indexed yet (${scopeLabel})`, { scope: scopeLabel }), - t("status.clickToScan", "Click the dot to start a scan") + [t("status.clickToScan", "Click the dot to start a scan"), watcherLine].filter(Boolean).join(" • ") ); } else { // Assets indexed - green @@ -476,6 +561,7 @@ export async function updateStatus(statusDot, statusText, capabilitiesSection = t("status.assetsIndexed", `${totalAssets.toLocaleString()} assets indexed (${scopeLabel})`, { count: totalAssets.toLocaleString(), scope: scopeLabel }), t("status.imagesVideos", `Images: ${counters.images || 0} - Videos: ${counters.videos || 0}`, { images: counters.images || 0, videos: counters.videos || 0 }), t("status.withWorkflows", `With workflows: ${withWorkflows} - Generation data: ${withGenerationData}`, { workflows: withWorkflows, gendata: withGenerationData }), + watcherLine, ], t("status.lastScan", `Last scan: ${lastScanText}`, { date: lastScanText }) ); diff --git a/js/features/viewer/genInfo.js b/js/features/viewer/genInfo.js index 445efd8f..a3d41c5f 100644 --- a/js/features/viewer/genInfo.js +++ b/js/features/viewer/genInfo.js @@ -10,6 +10,71 @@ const safeCall = (fn, fallback = null) => { return res === undefined ? fallback : res; }; +const GENINFO_CACHE_TTL_MS = APP_CONFIG?.VIEWER_GENINFO_TTL_MS ?? 30_000; +const GENINFO_ERROR_TTL_MS = APP_CONFIG?.VIEWER_GENINFO_ERROR_TTL_MS ?? 8_000; +const GENINFO_CACHE_MAX = APP_CONFIG?.VIEWER_GENINFO_MAX_ENTRIES ?? 300; + +const GENINFO_CACHE = new Map(); // key -> { at, data } +const GENINFO_ERROR_CACHE = new Map(); // key -> { at, error } +const GENINFO_INFLIGHT = new Map(); // key -> Promise + +const _pruneCache = (cache, ttl, maxEntries) => { + try { + const now = Date.now(); + for (const [key, value] of cache.entries()) { + if (!value) { + cache.delete(key); + continue; + } + if (now - (value.at || 0) > ttl) { + cache.delete(key); + } + } + if (cache.size <= maxEntries) return; + const sorted = Array.from(cache.entries()).sort((a, b) => (a?.[1]?.at || 0) - (b?.[1]?.at || 0)); + const excess = cache.size - maxEntries; + for (let i = 0; i < excess; i++) { + const key = sorted[i]?.[0]; + if (key != null) cache.delete(key); + } + } catch {} +}; + +const _cacheGet = (cache, key, ttl) => { + try { + const v = cache.get(key); + if (!v) return null; + if (Date.now() - (v.at || 0) > ttl) { + cache.delete(key); + return null; + } + return v.data ?? null; + } catch { + return null; + } +}; + +const _cacheSet = (cache, key, data, ttl, maxEntries) => { + try { + cache.set(key, { at: Date.now(), data }); + _pruneCache(cache, ttl, maxEntries); + } catch {} +}; + +const _getCacheKey = (asset) => { + try { + const id = asset?.id; + if (id != null) return `id:${id}`; + const fp = getFilePath(asset); + if (fp) return `fp:${fp}`; + const name = String(asset?.filename || asset?.name || "").trim(); + const sub = String(asset?.subfolder || "").trim(); + const type = String(asset?.source || asset?.type || "output").trim().toLowerCase(); + if (name) return `name:${type}:${sub}:${name}`; + } catch {} + return null; +}; + const getGenInfoStatus = (asset) => { try { if (!asset || typeof asset !== "object") return null; @@ -222,8 +287,30 @@ export async function ensureViewerMetadataAsset( ) { if (!asset || typeof asset !== "object") return asset; const id = asset?.id ?? null; + const cacheKey = _getCacheKey(asset); let full = asset; + const cachedGen = cacheKey ? _cacheGet(GENINFO_CACHE, cacheKey, GENINFO_CACHE_TTL_MS) : null; + if (cachedGen && typeof cachedGen === "object") { + return { ...asset, ...cachedGen }; + } + + const cachedError = cacheKey ? _cacheGet(GENINFO_ERROR_CACHE, cacheKey, GENINFO_ERROR_TTL_MS) : null; + if (cachedError) { + try { setGenInfoStatus(full, cachedError); } catch {} + return full; + } + + if (cacheKey && GENINFO_INFLIGHT.has(cacheKey)) { + try { + const inflight = GENINFO_INFLIGHT.get(cacheKey); + if (inflight && typeof inflight.then === "function") { + return await inflight; + } + } catch {} + } + + const run = async () => { const cached = id != null ? safeCall(() => metadataCache?.getCached?.(id)?.data || null, null) : null; if (cached && typeof cached === "object") full = { ...asset, ...cached }; @@ -305,7 +392,22 @@ export async function ensureViewerMetadataAsset( setGenInfoStatus(full, lastError); } } + if (_hasUsefulGenOrWorkflowPayload(full) && cacheKey) { + _cacheSet(GENINFO_CACHE, cacheKey, full, GENINFO_CACHE_TTL_MS, GENINFO_CACHE_MAX); + } else if (lastError && cacheKey) { + _cacheSet(GENINFO_ERROR_CACHE, cacheKey, lastError, GENINFO_ERROR_TTL_MS, GENINFO_CACHE_MAX); + } return full; + }; + + if (cacheKey) { + const p = run().finally(() => { + try { GENINFO_INFLIGHT.delete(cacheKey); } catch {} + }); + GENINFO_INFLIGHT.set(cacheKey, p); + return await p; + } + return await run(); } export function buildViewerMetadataBlocks({ title, asset, ui } = {}) { diff --git a/js/utils/dom.js b/js/utils/dom.js new file mode 100644 index 00000000..630803bc --- /dev/null +++ b/js/utils/dom.js @@ -0,0 +1,26 @@ +/** + * DOM helpers + */ + +/** + * Safely call Element.closest on a potentially non-Element target. + * Returns null if target is not suitable or any error occurs. + * @param {any} target + * @param {string} selector + * @returns {Element|null} + */ +export function safeClosest(target, selector) { + if (!selector) return null; + try { + if (!target) return null; + if (target instanceof Element && typeof target.closest === "function") { + return target.closest(selector); + } + // Text nodes or other non-Element targets + const parent = target?.parentElement; + if (parent && typeof parent.closest === "function") { + return parent.closest(selector); + } + } catch {} + return null; +} diff --git a/js/utils/extractOutputFiles.js b/js/utils/extractOutputFiles.js index 3559e0d9..444d2086 100644 --- a/js/utils/extractOutputFiles.js +++ b/js/utils/extractOutputFiles.js @@ -1,20 +1,98 @@ export function extractOutputFiles(output) { const files = []; - const addItems = (items) => { - if (!Array.isArray(items)) return; - items.forEach((item) => { - if (!item || !item.filename) return; - files.push({ - filename: item.filename, - subfolder: item.subfolder || "", - type: item.type || "output", - }); - }); + const seen = new Set(); + const MAX_ITEMS = 500; + const MAX_DEPTH = 4; + const MAX_SCAN_ITEMS = 2000; + const visited = typeof WeakSet !== "undefined" ? new WeakSet() : null; + + const pushItem = (item) => { + if (!item) return; + const rawPath = item.path || item.filepath || item.fullpath || item.fullPath || item.full_path; + let filename = item.filename; + if (!filename && rawPath) { + const parts = String(rawPath).split(/[/\\\\]/); + filename = parts[parts.length - 1]; + } + if (!filename) return; + filename = String(filename); + if (!filename) return; + const subfolder = item.subfolder || item.sub_folder || item.subFolder || ""; + const type = item.type || item.output_type || "output"; + const key = `${type}|${subfolder}|${filename}`.toLowerCase(); + if (seen.has(key)) return; + seen.add(key); + const entry = { filename, subfolder, type }; + if (rawPath) entry.path = rawPath; + files.push(entry); }; - addItems(output?.images); - addItems(output?.gifs); - addItems(output?.videos); + const addItems = (items) => { + if (!Array.isArray(items)) return; + let scanned = 0; + for (const item of items) { + if (files.length >= MAX_ITEMS || scanned >= MAX_SCAN_ITEMS) break; + scanned += 1; + if (item && typeof item === "object" && "filename" in item) { + pushItem(item); + } else { + walk(item, 1); + } + } + }; + + const walk = (node, depth) => { + if (!node || depth > MAX_DEPTH || files.length >= MAX_ITEMS) return; + if (typeof node !== "object") return; + if (visited) { + if (visited.has(node)) return; + visited.add(node); + } + + if (Array.isArray(node)) { + let scanned = 0; + for (const item of node) { + if (files.length >= MAX_ITEMS || scanned >= MAX_SCAN_ITEMS) break; + scanned += 1; + if (item && typeof item === "object" && "filename" in item) { + pushItem(item); + } else { + walk(item, depth + 1); + } + } + return; + } + + // Common ComfyUI output keys + if (node.images || node.gifs || node.videos) { + addItems(node.images); + addItems(node.gifs); + addItems(node.videos); + } + + // Known wrappers + if (node.output) walk(node.output, depth + 1); + if (node.outputs) walk(node.outputs, depth + 1); + if (node.result) walk(node.result, depth + 1); + if (node.data) walk(node.data, depth + 1); + + // Shallow scan of object values for nested outputs + if (depth < MAX_DEPTH) { + for (const value of Object.values(node)) { + if (files.length >= MAX_ITEMS) break; + if (!value || typeof value !== "object") continue; + if (value.images || value.gifs || value.videos) { + addItems(value.images); + addItems(value.gifs); + addItems(value.videos); + continue; + } + walk(value, depth + 1); + } + } + }; + + walk(output, 0); return files; } diff --git a/js/utils/fastUpload.js b/js/utils/fastUpload.js index ea5ab1db..f4f35e74 100644 --- a/js/utils/fastUpload.js +++ b/js/utils/fastUpload.js @@ -5,22 +5,32 @@ /** * Fast upload image to ComfyUI input directory using native endpoint * @param {File} file - The image file to upload - * @returns {Promise} - Response from the upload endpoint + * @returns {Promise<{ok: boolean, data: any, error?: string, code?: string, status?: number}>} - Safe result */ export async function fastUploadImageToInput(file) { const fd = new FormData(); fd.append("image", file, file.name); - const res = await fetch("/upload/image", { method: "POST", body: fd }); - if (!res.ok) throw new Error(`Upload failed: ${res.status}`); - return await res.json(); // { name: "file.png", ... } + try { + const res = await fetch("/upload/image", { method: "POST", body: fd }); + if (!res.ok) { + return { ok: false, data: null, error: `Upload failed: ${res.status}`, code: "UPLOAD_FAILED", status: res.status }; + } + const data = await res.json().catch(() => null); + if (!data) { + return { ok: false, data: null, error: "Invalid response", code: "INVALID_RESPONSE", status: res.status }; + } + return { ok: true, data, status: res.status }; + } catch (err) { + return { ok: false, data: null, error: err?.message || String(err), code: "NETWORK_ERROR" }; + } } /** * Fast upload file to ComfyUI input directory using custom endpoint * @param {File} file - The file to upload (for video, etc.) * @param {string} purpose - Purpose of upload (e.g. "node_drop") - * @returns {Promise} - Response from the upload endpoint + * @returns {Promise<{ok: boolean, data: any, error?: string, code?: string, status?: number}>} - Safe result */ export async function fastUploadFileToInput(file, purpose = null) { const fd = new FormData(); @@ -31,16 +41,26 @@ export async function fastUploadFileToInput(file, purpose = null) { url += `?purpose=${encodeURIComponent(purpose)}`; } - const res = await fetch(url, { method: "POST", body: fd }); - if (!res.ok) throw new Error(`Upload failed: ${res.status}`); - return await res.json(); // { name: "file.mp4", ... } + try { + const res = await fetch(url, { method: "POST", body: fd }); + if (!res.ok) { + return { ok: false, data: null, error: `Upload failed: ${res.status}`, code: "UPLOAD_FAILED", status: res.status }; + } + const data = await res.json().catch(() => null); + if (!data) { + return { ok: false, data: null, error: "Invalid response", code: "INVALID_RESPONSE", status: res.status }; + } + return { ok: true, data, status: res.status }; + } catch (err) { + return { ok: false, data: null, error: err?.message || String(err), code: "NETWORK_ERROR" }; + } } /** * Upload a file with appropriate method based on type * @param {File} file - The file to upload * @param {string} purpose - Purpose of upload (e.g. "node_drop") - * @returns {Promise} - Response from the upload endpoint + * @returns {Promise<{ok: boolean, data: any, error?: string, code?: string, status?: number}>} - Safe result */ export async function smartUploadToInput(file, purpose = null) { // Check if it's an image file @@ -54,4 +74,4 @@ export async function smartUploadToInput(file, purpose = null) { // Use custom endpoint for other files (videos, etc.) return await fastUploadFileToInput(file, purpose); } -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index b9922c87..3f1d856e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ name = "majoor-assetsmanager" description = "Majoor Assets Manager is an advanced asset browser for ComfyUI that provides a comprehensive solution for managing, organizing, and viewing your generated assets. It integrates directly into ComfyUI, offering features like full-text search, metadata extraction, rating and tagging systems, and advanced viewing capabilities." version = "2.2.3" +requires-python = ">=3.10, <3.15" license = {file = "LICENSE"} authors = [ { name = "MajoorWaldi" }, @@ -28,7 +29,6 @@ dependencies = [ "aiosqlite>=0.20.0", "pillow>=11.0.0", "send2trash>=1.8.0", - "watchdog>=2.1.0", "pywin32>=300; sys_platform == 'win32'", # Note: tkinter is required for native folder browser functionality # tkinter is included in standard Python distribution but may be missing in minimal installs diff --git a/requirements.txt b/requirements.txt index 50f8b688..a9b01337 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,13 +13,12 @@ pillow>=11.0.0 # Safe file deletion (recycle bin support) send2trash>=1.8.0 -# File system monitoring for auto-reindexing -watchdog>=2.1.0 +# File system monitoring for auto-reindexing (optional, enable with MJR_ENABLE_WATCHER=1) +watchdog>=3.0.0 # Windows-specific: Windows Property System metadata support # Only required on Windows; gracefully skipped on other platforms pywin32>=300; sys_platform == 'win32' -pywin32>=300; sys_platform == 'win32' # Note: tkinter is required for native folder browser functionality # tkinter is included in standard Python distribution but may be missing in minimal installs diff --git a/shared/scan_throttle.py b/shared/scan_throttle.py index a94cb3d3..4e9828a8 100644 --- a/shared/scan_throttle.py +++ b/shared/scan_throttle.py @@ -13,6 +13,7 @@ from backend.config import MANUAL_BG_SCAN_GRACE_SECONDS _LOCK = threading.Lock() _MANUAL_SCAN_TIMES: dict[str, float] = {} +_RECENT_SCAN_TIMES: dict[str, float] = {} _MAX_ENTRY_AGE = max(600, int(MANUAL_BG_SCAN_GRACE_SECONDS * 5)) @@ -54,11 +55,28 @@ def mark_directory_indexed( _cleanup_locked(now) +def mark_directory_scanned( + directory: str, + source: str = "output", + root_id: Optional[str] = None, +) -> None: + """ + Record that a background scan was scheduled/executed, regardless of metadata completeness. + Used to avoid repeated fast scans when the DB is empty. + """ + now = time.time() + key = _scan_key(directory, source, root_id) + with _LOCK: + _RECENT_SCAN_TIMES[key] = now + _cleanup_locked(now) + + def should_skip_background_scan( directory: str, source: str = "output", root_id: Optional[str] = None, grace_seconds: float = MANUAL_BG_SCAN_GRACE_SECONDS, + include_recent: bool = False, ) -> bool: """ Return True if a recent manual index was recorded for the same directory/source/root. @@ -68,11 +86,17 @@ def should_skip_background_scan( with _LOCK: ts = _MANUAL_SCAN_TIMES.get(key) if ts is None: - return False + if include_recent: + ts = _RECENT_SCAN_TIMES.get(key) + if ts is None: + return False + else: + return False if now - ts < grace_seconds: return True if now - ts > _MAX_ENTRY_AGE: - del _MANUAL_SCAN_TIMES[key] + _MANUAL_SCAN_TIMES.pop(key, None) + _RECENT_SCAN_TIMES.pop(key, None) return False @@ -82,3 +106,6 @@ def _cleanup_locked(now: float) -> None: keys_to_remove = [k for k, v in _MANUAL_SCAN_TIMES.items() if v < cutoff] for k in keys_to_remove: _MANUAL_SCAN_TIMES.pop(k, None) + keys_to_remove = [k for k, v in _RECENT_SCAN_TIMES.items() if v < cutoff] + for k in keys_to_remove: + _RECENT_SCAN_TIMES.pop(k, None) diff --git a/tests/README.md b/tests/README.md index 4a81eaa9..2443eb6b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -58,7 +58,6 @@ tests/ │ ├── test_collections_service.py │ ├── test_batch_zip.py │ ├── test_fast_scan.py -│ ├── test_watcher_restart.py │ ├── test_date_histogram.py │ └── test_viewer_grid_mask.py │ @@ -154,7 +153,6 @@ pytest tests/ --cov=backend --cov-report=html - Collections CRUD - Multi-file ZIP export -- File watcher - Date histogram - Viewer grid masks diff --git a/tests/config/test_comfy_output.py b/tests/config/test_comfy_output.py index c49ffa2f..77f250da 100644 --- a/tests/config/test_comfy_output.py +++ b/tests/config/test_comfy_output.py @@ -25,7 +25,9 @@ async def test_comfy_output(tmp_path): # Build services db_path = tmp_path / "comfy_output_test.db" - services = await build_services(str(db_path)) + services_res = await build_services(str(db_path)) + assert services_res.ok, services_res.error + services = services_res.data index_service = services["index"] # ComfyUI output directory. # Prefer explicit env override to avoid depending on local ComfyUI install. diff --git a/tests/conftest.py b/tests/conftest.py index 26023cf0..69584525 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,9 @@ async def services(tmp_path): db_path = str(tmp_path / "test_services.db") # build_services is now async - svc = await build_services(db_path) + svc_res = await build_services(db_path) + assert svc_res.ok, svc_res.error + svc = svc_res.data try: yield svc finally: diff --git a/tests/core/test_index.py b/tests/core/test_index.py index 1d4d64ed..fae1d1de 100644 --- a/tests/core/test_index.py +++ b/tests/core/test_index.py @@ -26,7 +26,9 @@ async def test_index_service(tmp_path): # Build services db_path = tmp_path / "test_index.db" - services = await build_services(str(db_path)) + services_res = await build_services(str(db_path)) + assert services_res.ok, services_res.error + services = services_res.data index_service = services["index"] parser_dir = Path(__file__).parent.parent / "parser" logger.info(f"\nTest 1: Scanning {parser_dir}") diff --git a/tests/core/test_integration.py b/tests/core/test_integration.py index 5f823b9f..4db1d696 100644 --- a/tests/core/test_integration.py +++ b/tests/core/test_integration.py @@ -118,7 +118,11 @@ def main(): try: # Build services (manual run uses direct build) from backend.deps import build_services - services = build_services("test_integration.db") + import asyncio + services_res = asyncio.run(build_services("test_integration.db")) + if not services_res.ok: + raise RuntimeError(services_res.error or "Failed to build services") + services = services_res.data print() # Test health diff --git a/tests/features/test_fast_scan.py b/tests/features/test_fast_scan.py index eacab1ff..45911337 100644 --- a/tests/features/test_fast_scan.py +++ b/tests/features/test_fast_scan.py @@ -11,7 +11,9 @@ async def test_fast_scan_enriches_metadata_in_background(tmp_path: Path): from backend.deps import build_services db_path = tmp_path / "fast_scan.db" - services = await build_services(db_path=str(db_path)) + services_res = await build_services(db_path=str(db_path)) + assert services_res.ok, services_res.error + services = services_res.data index = services["index"] db = services["db"] diff --git a/tests/features/test_open_in_folder.py b/tests/features/test_open_in_folder.py new file mode 100644 index 00000000..0cacb663 --- /dev/null +++ b/tests/features/test_open_in_folder.py @@ -0,0 +1,73 @@ +import json +from pathlib import Path + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +from backend.adapters.db.sqlite import Sqlite +from backend.routes.handlers.assets import register_asset_routes +from backend.shared import Result + + +@pytest.mark.asyncio +async def test_open_in_folder_returns_ok(tmp_path: Path, monkeypatch): + db = Sqlite(str(tmp_path / "assets.db")) + await db.aexecutescript( + """ + CREATE TABLE assets ( + id INTEGER PRIMARY KEY, + filepath TEXT NOT NULL + ); + """ + ) + + out_dir = tmp_path / "out" + out_dir.mkdir(parents=True, exist_ok=True) + file_path = out_dir / "a.png" + file_path.write_bytes(b"x") + await db.aexecute("INSERT INTO assets(id, filepath) VALUES (?, ?)", (1, str(file_path))) + + import backend.routes.handlers.assets as assets_mod + + async def _mock_require_services(): + return ({"db": db}, None) + + async def _mock_resolve_security_prefs(_svc): + return {} + + monkeypatch.setattr(assets_mod, "_require_services", _mock_require_services) + monkeypatch.setattr(assets_mod, "_resolve_security_prefs", _mock_resolve_security_prefs) + monkeypatch.setattr(assets_mod, "_require_operation_enabled", lambda *a, **k: Result.Ok(True)) + monkeypatch.setattr(assets_mod, "_require_write_access", lambda *a, **k: Result.Ok(True)) + monkeypatch.setattr(assets_mod, "_check_rate_limit", lambda *a, **k: (True, None)) + monkeypatch.setattr(assets_mod, "_is_path_allowed", lambda p: True) + monkeypatch.setattr(assets_mod, "_is_path_allowed_custom", lambda p: False) + + class _DummyPopen: + def __init__(self, *args, **kwargs): + pass + + monkeypatch.setattr(assets_mod.subprocess, "Popen", _DummyPopen) + + routes = web.RouteTableDef() + register_asset_routes(routes) + app = web.Application() + app.add_routes(routes) + + client = TestClient(TestServer(app)) + await client.start_server() + try: + resp = await client.post( + "/mjr/am/open-in-folder", + data=json.dumps({"asset_id": 1}), + headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"}, + ) + payload = await resp.json() + assert payload["ok"] is True, payload + finally: + await client.close() + try: + await db.aclose() + except Exception: + pass diff --git a/tests/features/test_upload_input.py b/tests/features/test_upload_input.py new file mode 100644 index 00000000..f89f5256 --- /dev/null +++ b/tests/features/test_upload_input.py @@ -0,0 +1,42 @@ +import pytest +from pathlib import Path +from aiohttp import web, FormData +from aiohttp.test_utils import TestClient, TestServer + +from backend.routes.handlers.scan import register_scan_routes + + +@pytest.mark.asyncio +async def test_upload_input_file_writes_to_input_dir(tmp_path: Path, monkeypatch): + import backend.routes.handlers.scan as scan_mod + + input_dir = tmp_path / "input" + input_dir.mkdir(parents=True, exist_ok=True) + + class _FolderPathsStub: + @staticmethod + def get_input_directory() -> str: + return str(input_dir) + + monkeypatch.setattr(scan_mod, "folder_paths", _FolderPathsStub(), raising=False) + + routes = web.RouteTableDef() + register_scan_routes(routes) + app = web.Application() + app.add_routes(routes) + + client = TestClient(TestServer(app)) + await client.start_server() + try: + data = FormData() + data.add_field("file", b"hello-world", filename="hello.txt", content_type="text/plain") + resp = await client.post( + "/mjr/am/upload_input?purpose=node_drop", + data=data, + headers={"X-Requested-With": "XMLHttpRequest"}, + ) + payload = await resp.json() + assert payload["ok"] is True, payload + assert (input_dir / "hello.txt").exists() + finally: + await client.close() diff --git a/tests/features/test_watcher_restart.py b/tests/features/test_watcher_restart.py deleted file mode 100644 index 0c4ba8c9..00000000 --- a/tests/features/test_watcher_restart.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -import time - - -@pytest.mark.asyncio -async def test_directory_watcher_can_restart_same_instance(): - from backend.features.index.watcher import DirectoryWatcher - from backend.shared import Result - - class DummyIndexService: - def scan_directory(self, *args, **kwargs): - return Result.Ok(True) - - watcher = DirectoryWatcher( - index_service=DummyIndexService(), - directories=[], - interval_seconds=0.01, - join_timeout=0.2, - ) - - watcher.start() - assert watcher.is_alive() - time.sleep(0.03) - watcher.stop() - assert not watcher.is_alive() - - # Restart should not raise RuntimeError (Thread started twice) - watcher.start() - assert watcher.is_alive() - watcher.stop() - assert not watcher.is_alive() - diff --git a/tests/features/test_watcher_routes.py b/tests/features/test_watcher_routes.py new file mode 100644 index 00000000..895ca9b0 --- /dev/null +++ b/tests/features/test_watcher_routes.py @@ -0,0 +1,64 @@ +import json + +import pytest +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +from backend.routes.handlers.scan import register_scan_routes + +@pytest.mark.asyncio +async def test_watcher_scope_returns_ok_without_watcher(monkeypatch): + import backend.routes.handlers.scan as scan_mod + + async def _mock_require_services(): + return ({"watcher": None, "db": None}, None) + + monkeypatch.setattr(scan_mod, "_require_services", _mock_require_services) + + routes = web.RouteTableDef() + register_scan_routes(routes) + app = web.Application() + app.add_routes(routes) + + client = TestClient(TestServer(app)) + await client.start_server() + try: + resp = await client.post( + "/mjr/am/watcher/scope", + data=json.dumps({"scope": "output"}), + headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"}, + ) + payload = await resp.json() + assert payload["ok"] is True, payload + assert payload["data"]["enabled"] is False + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_watcher_toggle_requires_index(monkeypatch): + import backend.routes.handlers.scan as scan_mod + + async def _mock_require_services(): + return ({"watcher": None}, None) + + monkeypatch.setattr(scan_mod, "_require_services", _mock_require_services) + + routes = web.RouteTableDef() + register_scan_routes(routes) + app = web.Application() + app.add_routes(routes) + + client = TestClient(TestServer(app)) + await client.start_server() + try: + resp = await client.post( + "/mjr/am/watcher/toggle", + data=json.dumps({"enabled": True}), + headers={"Content-Type": "application/json", "X-Requested-With": "XMLHttpRequest"}, + ) + payload = await resp.json() + assert payload["ok"] is False, payload + assert payload.get("code") == "SERVICE_UNAVAILABLE" + finally: + await client.close() diff --git a/tests/regressions/test_scan_batching.py b/tests/regressions/test_scan_batching.py index cdbb8aec..7e0b81f6 100644 --- a/tests/regressions/test_scan_batching.py +++ b/tests/regressions/test_scan_batching.py @@ -14,7 +14,9 @@ async def test_scan_uses_batched_transactions(tmp_path: Path): from backend.deps import build_services db_path = tmp_path / "batching.db" - services = await build_services(db_path=str(db_path)) + services_res = await build_services(db_path=str(db_path)) + assert services_res.ok, services_res.error + services = services_res.data index = services["index"] db = services["db"] diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..704885fe --- /dev/null +++ b/tox.ini @@ -0,0 +1,10 @@ +[tox] +envlist = py310, py311 +skipsdist = True + +[testenv] +deps = + -rrequirements.txt + pytest +commands = + python -m pytest -q