* feat(activity): scan new and unresolved files by default, all files on request
Scan for metadata now skips, before reading any metadata, the files the
Library already links with a resolved link (one generated_images row with
exactly one pair link), found with one streamed query that keeps only the
scanned paths and compares paths with os.path.normcase on both sides.
Skipped files count as Already known ("Skipped · already linked").
Unlinked, fallback-linked, ambiguous and unresolved files are read again.
POST /prompt_manager/jobs/scan takes {scope: "new" | "all"} (default
"new", anything else is a 400). The review dialog shows the concept's
"New and unresolved files" and adds an "All files" switch (D58). The
legacy /prompt_manager/scan route keeps scanning all files with unchanged
SSE payloads.
* feat(activity): download a job report
GET /prompt_manager/jobs/{id}/report returns the summary, counters, every
outcome group's listed rows with truncated flags, the scope and the log as a
JSON attachment (prompt-manager-<kind>-<YYYYMMDD-HHMM>.json), with no absolute
paths. The job view's Download report saves it through the api-client URL, and
the scan log opens at its newest line and follows new lines only while the
reader is at the bottom.
* fix(activity): re-read known files whose size changed, and review follow-ups
The new scope now skips a known, resolved file only while its stored
generated_images.file_size equals its size on disk; a NULL stored size is
re-read. A file replaced at the same path is read again and its conflict
shows under Needs review. The scan stores each file's size when it ingests
it, and both sides of the known-set match go through normalize_image_path.
A failed known-set read logs and falls back to reading every file, and an
empty request body asks for the default scope. The All files switch is
described by its hint, a lost job's Download report is disabled, the report
toast is skipped once the view is gone, and the scan log opens at its
newest line every time. Fixtures use the real scope shape.
* test(activity): wait for the job view's settled detail read, not a fixed delay
The 200-row cap test now starts counting detail reads once one read has
returned the full listing and none is in flight. The job view re-reads after
an in-flight read only when the summary still lists more than that read
returned, so no redundant read is left due after opening.
162 lines
5.8 KiB
Python
162 lines
5.8 KiB
Python
"""Session job routes (Activity): list, inspect, follow and cancel jobs.
|
|
|
|
The registry itself is ``py/api/jobs.py``; the scan job's runner is in
|
|
``py/api/admin.py``. Every response leaves out absolute server paths: job
|
|
rows already carry root-relative paths and public root labels.
|
|
"""
|
|
|
|
import contextlib
|
|
import json
|
|
|
|
from aiohttp import web
|
|
|
|
from .jobs import DEFAULT_SCAN_SCOPE, SCAN_SCOPES, JobBusy
|
|
|
|
|
|
def _not_found():
|
|
return web.json_response({"success": False, "error": "Job not found"}, status=404)
|
|
|
|
|
|
def _bad_request(message):
|
|
return web.json_response({"success": False, "error": message}, status=400)
|
|
|
|
|
|
async def _scan_scope(request):
|
|
"""The requested scan scope ("new" or "all"), or None when the body is invalid.
|
|
|
|
An empty body (also a chunked one with no bytes), or one without
|
|
``scope``, asks for the default.
|
|
"""
|
|
if not request.can_read_body:
|
|
return DEFAULT_SCAN_SCOPE
|
|
text = await request.text()
|
|
if not text.strip(): # e.g. a chunked POST that sent no bytes
|
|
return DEFAULT_SCAN_SCOPE
|
|
try:
|
|
body = json.loads(text)
|
|
except ValueError:
|
|
return None
|
|
if not isinstance(body, dict):
|
|
return None
|
|
scope = body.get("scope", DEFAULT_SCAN_SCOPE)
|
|
return scope if scope in SCAN_SCOPES else None
|
|
|
|
|
|
def _summary_event(job):
|
|
return f"data: {json.dumps(job.summary())}\n\n".encode("utf-8")
|
|
|
|
|
|
class JobRoutesMixin:
|
|
"""Mixin providing the /prompt_manager/jobs endpoints."""
|
|
|
|
def _register_job_routes(self, routes):
|
|
@routes.get("/prompt_manager/jobs")
|
|
async def list_jobs_route(request):
|
|
return await self.list_jobs(request)
|
|
|
|
@routes.post("/prompt_manager/jobs/scan")
|
|
async def start_scan_job_route(request):
|
|
return await self.start_scan_job(request)
|
|
|
|
@routes.get("/prompt_manager/jobs/{job_id}")
|
|
async def job_detail_route(request):
|
|
return await self.job_detail(request)
|
|
|
|
@routes.get("/prompt_manager/jobs/{job_id}/report")
|
|
async def job_report_route(request):
|
|
return await self.job_report(request)
|
|
|
|
@routes.get("/prompt_manager/jobs/{job_id}/events")
|
|
async def job_events_route(request):
|
|
return await self.job_events(request)
|
|
|
|
@routes.post("/prompt_manager/jobs/{job_id}/cancel")
|
|
async def cancel_job_route(request):
|
|
return await self.cancel_job(request)
|
|
|
|
async def list_jobs(self, request):
|
|
"""This session's jobs: running first, then finished, newest first."""
|
|
jobs = [job.summary() for job in self._jobs.list()]
|
|
return web.json_response({"success": True, "jobs": jobs})
|
|
|
|
async def job_detail(self, request):
|
|
job = self._jobs.get(request.match_info["job_id"])
|
|
if job is None:
|
|
return _not_found()
|
|
return web.json_response({"success": True, "job": job.detail()})
|
|
|
|
async def job_report(self, request):
|
|
"""The job's whole record as a JSON attachment (no absolute paths)."""
|
|
job = self._jobs.get(request.match_info["job_id"])
|
|
if job is None:
|
|
return _not_found()
|
|
body = json.dumps(job.report(), indent=2)
|
|
disposition = f'attachment; filename="{job.report_filename()}"'
|
|
return web.Response(
|
|
text=body,
|
|
content_type="application/json",
|
|
headers={
|
|
"Content-Disposition": disposition,
|
|
"Cache-Control": "no-store",
|
|
},
|
|
)
|
|
|
|
async def cancel_job(self, request):
|
|
"""Stop a job at its next file boundary; a finished job is a no-op."""
|
|
job_id = request.match_info["job_id"]
|
|
if self._jobs.get(job_id) is None:
|
|
return _not_found()
|
|
requested = self._jobs.cancel(job_id)
|
|
return web.json_response({"success": True, "cancellation_requested": requested})
|
|
|
|
async def start_scan_job(self, request):
|
|
"""Start the metadata scan as a job (202), or 409 naming the busy job.
|
|
|
|
The body's ``scope`` is "new" (the default: new and unresolved files)
|
|
or "all"; anything else is a 400.
|
|
"""
|
|
files = await _scan_scope(request)
|
|
if files is None:
|
|
return _bad_request('The scan scope must be "new" or "all"')
|
|
try:
|
|
job = self._start_scan_job(files)
|
|
except JobBusy as busy:
|
|
running = busy.running
|
|
return web.json_response(
|
|
{
|
|
"success": False,
|
|
"code": "busy",
|
|
"error": str(busy),
|
|
"running": {"id": running.id, "title": running.title},
|
|
},
|
|
status=409,
|
|
)
|
|
return web.json_response({"success": True, "job": job.summary()}, status=202)
|
|
|
|
async def job_events(self, request):
|
|
"""Stream the job's summary as SSE: now, after each event, and at the end."""
|
|
job = self._jobs.get(request.match_info["job_id"])
|
|
if job is None:
|
|
return _not_found()
|
|
response = web.StreamResponse(
|
|
status=200,
|
|
reason="OK",
|
|
headers={
|
|
"Content-Type": "text/event-stream",
|
|
"Cache-Control": "no-cache",
|
|
"Connection": "keep-alive",
|
|
},
|
|
)
|
|
await response.prepare(request)
|
|
try:
|
|
await response.write(_summary_event(job))
|
|
async with contextlib.aclosing(job.events()) as events:
|
|
async for _payload in events:
|
|
await response.write(_summary_event(job))
|
|
# The last event is published before the job takes its final state.
|
|
await response.write(_summary_event(job))
|
|
await response.write_eof()
|
|
except Exception as e: # the socket closed under us; the job goes on
|
|
self.logger.info(f"Job progress client disconnected ({e})")
|
|
return response
|