Merge remote-tracking branch 'origin/main' into fix/scan-nan-workflow

py/api/__init__.py

@@ -8,6 +8,7 @@ Split into domain-specific mixins for maintainability:
 - AutotagRoutesMixin: auto-tagging model management and tagging
 - JobRoutesMixin: the session job registry behind Activity (py/api/jobs.py)
 - ImageTagJobsMixin: bulk Add tags / Auto-tag as image-tags jobs
+- SystemRoutesMixin: System health check, Overview and usage statistics
 """
 
 import asyncio
@@ -36,6 +37,7 @@ from .admin import AdminRoutesMixin
 from .settings_routes import SettingsRoutesMixin
 from .job_routes import JobRoutesMixin
 from .image_tag_jobs import ImageTagJobsMixin
+from .system import SystemRoutesMixin
 from .jobs import JobRegistry
 from .logging_routes import LoggingRoutesMixin
 from .autotag_routes import AutotagRoutesMixin
@@ -270,6 +272,7 @@ class PromptManagerAPI(
     SettingsRoutesMixin,
     JobRoutesMixin,
     ImageTagJobsMixin,
+    SystemRoutesMixin,
     LoggingRoutesMixin,
     AutotagRoutesMixin,
     LoraIntegrationMixin,
@@ -326,6 +329,11 @@ class PromptManagerAPI(
         self._scan_job = None
         # The current duplicate scan (a py/api/jobs.Job outside the registry).
         self._duplicate_job = None
+        # The last System health report this session and the database
+        # generation it describes, bumped when a restore replaces self.db
+        # (py/api/system.py).
+        self._health_report = None
+        self._db_generation = 0
 
         # Run cleanup on initialization to remove any existing duplicates
         try:
@@ -483,6 +491,7 @@ class PromptManagerAPI(
         self._register_admin_routes(routes)
         self._register_settings_routes(routes)
         self._register_job_routes(routes)
+        self._register_system_routes(routes)
         self._register_logging_routes(routes)
         self._register_autotag_routes(routes)
         self._register_lora_routes(routes)
@@ -729,6 +738,13 @@ class PromptManagerAPI(
         Returns:
             List[Path]: Validated directory paths, possibly empty.
         """
+        return self._output_dirs_with_origin()[0]
+
+    def _output_dirs_with_origin(self):
+        """``(dirs, auto_detected)``: the ``_get_all_output_dirs`` list, and
+        whether it comes from detection or ComfyUI's output rather than from
+        a valid configured root (False when there is no directory at all).
+        """
         from ..config import GalleryConfig
 
         configured = list(GalleryConfig.MONITORING_DIRECTORIES)
@@ -747,15 +763,16 @@ class PromptManagerAPI(
             output_dirs.append(Path(real))
 
         if output_dirs:
-            return output_dirs
+            return output_dirs, False
 
         if configured:
             # Every configured root was rejected: never let auto-detection
             # resurrect it, serve ComfyUI's own output directory instead.
-            return self._comfyui_output_dir_list()
-
-        fallback = self._find_comfyui_output_dir()
-        return [Path(fallback)] if fallback else []
+            fallback_dirs = self._comfyui_output_dir_list()
+        else:
+            fallback = self._find_comfyui_output_dir()
+            fallback_dirs = [Path(fallback)] if fallback else []
+        return fallback_dirs, bool(fallback_dirs)
 
     def _comfyui_output_dir_list(self):
         """ComfyUI's output directory as a one-element list, or [] if unknown."""

py/api/admin.py

@@ -41,7 +41,7 @@ try:
         PLACEHOLDERS,
         GenerationParamsError,
         extract_generation_params,
-        parse_js_json,
+        parse_page_json,
     )
     from ...utils.prompt_pair_resolver import resolve_prompt_pair
 except ImportError:
@@ -51,7 +51,7 @@ except ImportError:
         PLACEHOLDERS,
         GenerationParamsError,
         extract_generation_params,
-        parse_js_json,
+        parse_page_json,
     )
     from utils.prompt_pair_resolver import resolve_prompt_pair
 
@@ -804,6 +804,9 @@ class AdminRoutesMixin:
             result = await self._run_in_executor(
                 self._delete_duplicate_images_sync, image_paths, output_dir
             )
+            if result["deleted_count"]:
+                # The cached listing would keep (and count) the deleted files.
+                self.invalidate_gallery_cache()
 
             response_data = {
                 "success": True,
@@ -1271,6 +1274,7 @@ class AdminRoutesMixin:
                 self.logger.info(f"Removed {len(removed)} old safety backups")
             previous = self.db
             self.db = PromptDatabase(model.db_path)
+            self._database_replaced()  # a health report described the old one
             # Its connections would otherwise linger (a file lock on Windows)
             previous.close_all()
             prompt_count = self.db.model.get_database_info().get("total_prompts", 0)
@@ -1602,11 +1606,14 @@ class AdminRoutesMixin:
 
     @staticmethod
     def _graph_prompt_text(metadata):
-        """The positive prompt the browser parser traces from the graphs, or None."""
+        """The positive prompt the browser parser traces from the graphs, or None.
+
+        Records parse as the legacy pages parsed them (NaN reads as null).
+        """
         records = {}
         for key in ("prompt", "workflow"):
             value = metadata.get(key)
-            records[key] = parse_js_json(value) if isinstance(value, str) else value
+            records[key] = parse_page_json(value) if isinstance(value, str) else value
         try:
             positive = extract_generation_params(records)["positivePrompt"]
         except GenerationParamsError:

py/api/jobs.py

@@ -88,6 +88,9 @@ DUPLICATE_CLEANUP_JOB_KIND = "duplicate_cleanup"
 THUMBNAILS_JOB_KIND = "thumbnails"
 LORA_IMPORT_JOB_KIND = "lora_import"
 AUTOTAG_BATCH_JOB_KIND = "autotag"
+# The System health check (py/api/system.py): a read-only ``read`` job with
+# no file outcomes, so it may run beside a scan.
+HEALTH_JOB_KIND = "health"
 
 
 def resolve_admission(admission=None, record_changing=None) -> str:

py/api/system.py

@@ -0,0 +1,207 @@
+"""System routes: the read-only health check, the Overview and usage statistics.
+
+- ``POST /prompt_manager/system/health`` starts the health check
+  (``database/health.py``) off-loop as a ``read`` job of kind ``health``: one
+  at a time, beside a scan, not cancellable and with no file outcomes. The
+  API keeps the last report for this session; ``GET`` returns it or null.
+  A report describes the database it was taken from: replacing the database
+  (a restore) forgets it, and a check that started before the swap is not
+  kept.
+- ``GET /prompt_manager/system/overview`` is the System › Overview aggregate.
+  Every value is real or null: there is no backup status until backups report
+  one, no health result until a check ran, and no Needs-review count until
+  the review queue exists. The missing-thumbnail count is not part of it:
+  the page reads ``/images/thumbnails/status``, which walks every source.
+- ``GET /prompt_manager/system/statistics`` lists runs and ratings per
+  category, with the legacy ``/stats`` totals and the Overview's count of
+  rated images (images whose linked prompt is rated).
+
+New routes answer errors with ``{success:false, code, error}``.
+"""
+
+import asyncio
+import os
+from collections import namedtuple
+
+from aiohttp import web
+
+from .job_routes import busy_response
+from .jobs import HEALTH_JOB_KIND, JobBusy
+
+HEALTH_JOB_TITLE = "Health check"
+_HEALTH_STATUS_LINES = {
+    "passed": "Health check passed",
+    "issues": "Health check found issues",
+    "incomplete": "Health check incomplete",
+}
+
+
+# The last health report and the database generation it was taken from.
+_StoredReport = namedtuple("_StoredReport", "generation report")
+
+
+def _failure(message, status=500, code=None):
+    return web.json_response(
+        {"success": False, "code": code, "error": message}, status=status
+    )
+
+
+def health_summary(report):
+    """What the Overview shows of a report: when, the outcome, checks with issues."""
+    if report is None:
+        return None
+    return {
+        "checked_at": report["checked_at"],
+        "status": report["status"],
+        "issues": sum(1 for check in report["checks"] if check["status"] == "issues"),
+    }
+
+
+class SystemRoutesMixin:
+    """Mixin providing the /prompt_manager/system endpoints."""
+
+    def _register_system_routes(self, routes):
+        @routes.post("/prompt_manager/system/health")
+        async def start_health_check_route(request):
+            return await self.start_health_check(request)
+
+        @routes.get("/prompt_manager/system/health")
+        async def health_report_route(request):
+            return await self.get_health_report(request)
+
+        @routes.get("/prompt_manager/system/overview")
+        async def system_overview_route(request):
+            return await self.get_system_overview(request)
+
+        @routes.get("/prompt_manager/system/statistics")
+        async def system_statistics_route(request):
+            return await self.get_system_statistics(request)
+
+    # ── health ────────────────────────────────────────────────────────
+
+    def _health_roots(self):
+        """The sources missing files are judged under.
+
+        The effective sources, plus every configured source that is not an
+        existing directory (an unmounted drive): the check then reports
+        missing files as unavailable instead of counting that source's links.
+        """
+        from ..config import GalleryConfig
+
+        roots = [str(path) for path in self._get_all_output_dirs()]
+        for configured in GalleryConfig.MONITORING_DIRECTORIES:
+            try:
+                resolved = GalleryConfig.resolve_gallery_root(configured)
+            except (OSError, TypeError, ValueError):
+                continue
+            if not os.path.isdir(resolved):
+                roots.append(resolved)
+        return roots
+
+    def _database_replaced(self):
+        """Forget what described the previous database (call after swapping ``db``)."""
+        self._db_generation += 1
+        self._health_report = None
+
+    def _current_health_report(self):
+        """The last report, or None when none was kept for the current database."""
+        stored = self._health_report
+        generation = self._db_generation
+        if isinstance(stored, _StoredReport) and stored.generation == generation:
+            return stored.report
+        return None
+
+    async def start_health_check(self, request):
+        """Start the health check job (202), or 409 naming the running check."""
+        generation = self._db_generation
+        db = self.db
+        try:
+            roots = await self._run_in_executor(self._health_roots)
+        except Exception:
+            self.logger.exception("Health check could not resolve the image sources")
+            return _failure("Failed to start the health check")
+
+        async def run(job):
+            job.update(status_line="Checking the database and image links")
+            report = await self._run_in_executor(db.run_health_checks, roots)
+            if generation == self._db_generation:
+                self._health_report = _StoredReport(generation, report)
+            else:
+                self.logger.info("Health report dropped: the database was replaced")
+            job.update(progress=100, status_line=_HEALTH_STATUS_LINES[report["status"]])
+
+        try:
+            job = self._jobs.start(
+                HEALTH_JOB_KIND,
+                HEALTH_JOB_TITLE,
+                admission="read",
+                cancellable=False,
+                scope={},
+                runner=run,
+            )
+        except JobBusy as busy:
+            return busy_response(busy)
+        return web.json_response({"success": True, "job": job.summary()}, status=202)
+
+    async def get_health_report(self, request):
+        return web.json_response(
+            {"success": True, "report": self._current_health_report()}
+        )
+
+    # ── overview ──────────────────────────────────────────────────────
+
+    async def get_system_overview(self, request):
+        from ..config import GalleryConfig
+
+        try:
+            (output_dirs, auto_detected), counts = await asyncio.gather(
+                self._run_in_executor(self._output_dirs_with_origin),
+                self._run_in_executor(self.db.library_counts),
+            )
+        except Exception:
+            self.logger.exception("System overview failed")
+            return _failure("Failed to read the overview")
+        return web.json_response(
+            {
+                "success": True,
+                "library": {
+                    "positive_prompts": counts["positive_prompts"],
+                    "negative_variants": counts["negative_variants"],
+                },
+                "images": {
+                    "indexed": counts["images"],
+                    "rated": counts["rated_images"],
+                },
+                "health": health_summary(self._current_health_report()),
+                "backup": None,  # no verified backup status exists yet (S8)
+                "needs_review": {"pending": None},  # no review queue yet (Q6)
+                "sources": {
+                    "configured": len(GalleryConfig.MONITORING_DIRECTORIES),
+                    "effective": len(output_dirs),
+                    "auto_detected": auto_detected,
+                },
+            },
+            headers={"Cache-Control": "no-store"},
+        )
+
+    # ── statistics ────────────────────────────────────────────────────
+
+    async def get_system_statistics(self, request):
+        try:
+            by_category, totals, counts = await asyncio.gather(
+                self._run_in_executor(self.db.usage_by_category),
+                self._run_in_executor(self.db.get_statistics),
+                self._run_in_executor(self.db.library_counts),
+            )
+        except Exception:
+            self.logger.exception("System statistics failed")
+            return _failure("Failed to read statistics")
+        return web.json_response(
+            {
+                "success": True,
+                "by_category": by_category,
+                "totals": totals,
+                "rated_images": counts["rated_images"],
+            },
+            headers={"Cache-Control": "no-store"},
+        )