feat(activity): download a job report
GET /prompt_manager/jobs/{id}/report returns the summary, counters, every
outcome group's listed rows with truncated flags, the scope and the log as a
JSON attachment (prompt-manager-<kind>-<YYYYMMDD-HHMM>.json), with no absolute
paths. The job view's Download report saves it through the api-client URL, and
the scan log opens at its newest line and follows new lines only while the
reader is at the bottom.
This commit is contained in:
+12
@@ -649,6 +649,18 @@ Recorded shell deviations:
|
||||
place-dependent top margin; `workspace-activity.spec.mjs` covers the Scope value and
|
||||
the request both ways; `tests/test_scan_scope.py` covers the skip, the re-read of
|
||||
unresolved files, "all", the legacy route, the 400 and Windows case folding.
|
||||
- **D59** (job report and log, Activity-2 Task 5): the job view's "Download report" is
|
||||
enabled. It fetches `GET /prompt_manager/jobs/<id>/report` through the api-client URL
|
||||
(so a proxy mount works) and saves a JSON attachment named
|
||||
`prompt-manager-<kind>-<YYYYMMDD-HHMM>.json` (server local time), then toasts "Job
|
||||
report downloaded." (or "The report could not be downloaded. Try again."). The report
|
||||
holds the summary, every counter, each outcome group's exact `count`, its listed
|
||||
`rows` (at most 200) and a `truncated` flag, the scope and the log; rows and log lines
|
||||
carry root-relative paths only, never an absolute path (an unknown id is a 404).
|
||||
The scan log opens at its newest line and follows new lines only while the reader is
|
||||
at the bottom; a reader who scrolled up keeps the position.
|
||||
`test_report_is_attachment_without_absolute_paths` and the browser tests `download
|
||||
report saves the job JSON` and `log keeps the reader's scroll position` cover it.
|
||||
- **D29**: no prototype copy. There is no "Interactive concept" badge and no
|
||||
"What changes & why" button. The footer reads "ComfyAssets · Local tools. Your
|
||||
library.", and the title has no " concept" suffix. Save view's dialog action reads
|
||||
|
||||
@@ -58,6 +58,10 @@ class JobRoutesMixin:
|
||||
async def job_detail_route(request):
|
||||
return await self.job_detail(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/report")
|
||||
async def job_report_route(request):
|
||||
return await self.job_report(request)
|
||||
|
||||
@routes.get("/prompt_manager/jobs/{job_id}/events")
|
||||
async def job_events_route(request):
|
||||
return await self.job_events(request)
|
||||
@@ -77,6 +81,22 @@ class JobRoutesMixin:
|
||||
return _not_found()
|
||||
return web.json_response({"success": True, "job": job.detail()})
|
||||
|
||||
async def job_report(self, request):
|
||||
"""The job's whole record as a JSON attachment (no absolute paths)."""
|
||||
job = self._jobs.get(request.match_info["job_id"])
|
||||
if job is None:
|
||||
return _not_found()
|
||||
body = json.dumps(job.report(), indent=2)
|
||||
disposition = f'attachment; filename="{job.report_filename()}"'
|
||||
return web.Response(
|
||||
text=body,
|
||||
content_type="application/json",
|
||||
headers={
|
||||
"Content-Disposition": disposition,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
async def cancel_job(self, request):
|
||||
"""Stop a job at its next file boundary; a finished job is a no-op."""
|
||||
job_id = request.match_info["job_id"]
|
||||
|
||||
@@ -247,6 +247,36 @@ class Job:
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report(self):
|
||||
"""The downloadable report: the summary, every outcome group and the log.
|
||||
|
||||
Each group carries its exact count, its listed rows (at most
|
||||
MAX_OUTCOME_ROWS) and whether the list is cut short. Rows and log lines
|
||||
already hold root-relative paths only.
|
||||
"""
|
||||
summary = self.summary()
|
||||
summary.pop("listed", None)
|
||||
summary.pop("log_length", None)
|
||||
return {
|
||||
"report_version": 1,
|
||||
"generated_at": self._clock(),
|
||||
"job": summary,
|
||||
"outcomes": {
|
||||
group: {
|
||||
"count": self.counters.get(group, len(rows)),
|
||||
"rows": [dict(row) for row in rows],
|
||||
"truncated": self.counters.get(group, len(rows)) > len(rows),
|
||||
}
|
||||
for group, rows in self.outcomes.items()
|
||||
},
|
||||
"log": list(self.log),
|
||||
}
|
||||
|
||||
def report_filename(self):
|
||||
"""The download name: prompt-manager-KIND-YYYYMMDD-HHMM.json (local time)."""
|
||||
stamp = time.strftime("%Y%m%d-%H%M", time.localtime(self._clock()))
|
||||
return f"prompt-manager-{self.kind}-{stamp}.json"
|
||||
|
||||
|
||||
class JobRegistry:
|
||||
"""Every job of this server session, running and recently finished."""
|
||||
|
||||
@@ -449,11 +449,11 @@ test('job view counters follow the mapping and update live', async ({ page, requ
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 8240, counters: { added: 1420, known: 6511, review: 37, no_metadata: 254, errors: 18 } });
|
||||
await openJob(page, job.id);
|
||||
// The header: the title, the detail line, then Download report (Task 5) and Cancel scan (danger).
|
||||
// The header: the title, the detail line, then Download report (enabled in Task 5) and Cancel scan (danger).
|
||||
await expect(heading(page).locator('h1')).toHaveText('Scan for metadata');
|
||||
await expect(heading(page).locator('p.muted')).toHaveText(/^Started \d\d:\d\d · 2 output folders · closing this view does not cancel it$/);
|
||||
await expect(heading(page).locator('.heading-actions > button')).toHaveText(['Download report', 'Cancel scan']);
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeDisabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Download report', exact: true })).toBeEnabled();
|
||||
await expect(heading(page).getByRole('button', { name: 'Cancel scan', exact: true })).toHaveClass(/danger/);
|
||||
// Progress: the bar, the server's line and the five tiles in order and colour.
|
||||
await expect(progressSection(page).locator('.panel-heading > span.small')).toHaveText('Processing file 8,240 of 19,622');
|
||||
@@ -714,3 +714,62 @@ test('a finished job the server pruned leaves its open view honest', async ({ pa
|
||||
await expect(heading(page).locator('h1')).toBeFocused();
|
||||
await expect(page).toHaveTitle('Activity · Prompt Manager');
|
||||
});
|
||||
|
||||
// Activity-2 Task 5 (D59): Download report and the log's scroll position.
|
||||
test('download report saves the job JSON', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await step(request, job.id, { processed: 2, records: [
|
||||
{ group: 'added', path: 'sub/a.png', root: 'output', detail: 'Added' },
|
||||
{ group: 'errors', path: 'broken/b.png', root: 'output', detail: 'Error' },
|
||||
] });
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await expect(button).toBeEnabled();
|
||||
const [download] = await Promise.all([page.waitForEvent('download'), button.click()]);
|
||||
expect(download.suggestedFilename()).toMatch(/^prompt-manager-scan-\d{8}-\d{4}\.json$/);
|
||||
const path = await download.path();
|
||||
const report = JSON.parse(await (await import('node:fs/promises')).readFile(path, 'utf8'));
|
||||
expect(report.job.id).toBe(job.id);
|
||||
expect(report.outcomes.added.rows.map(row => row.path)).toEqual(['sub/a.png']);
|
||||
expect(report.outcomes.errors.truncated).toBe(false);
|
||||
expect(report.log.length).toBeGreaterThan(0);
|
||||
await expect(toast(page)).toHaveText('Job report downloaded.');
|
||||
});
|
||||
|
||||
test('log keeps the reader\'s scroll position', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
const more = (from, count) => Array.from({ length: count }, (_, index) => ({ group: 'known', path: `log/f_${from + index}.png`,
|
||||
root: 'output', detail: 'Known' }));
|
||||
await step(request, job.id, { processed: 60, records: more(0, 60) });
|
||||
await openJob(page, job.id);
|
||||
await scanLog(page).locator('summary').click();
|
||||
const pre = scanLog(page).locator('pre');
|
||||
await expect(pre).toContainText('log/f_59.png');
|
||||
const metrics = () => pre.evaluate(node => ({ top: node.scrollTop, max: node.scrollHeight - node.clientHeight }));
|
||||
// Opening starts at the newest line.
|
||||
await expect.poll(async () => { const m = await metrics(); return m.max > 0 && m.top >= m.max - 2; }).toBe(true);
|
||||
// Pinned to the bottom: new lines are followed.
|
||||
await step(request, job.id, { processed: 90, records: more(60, 30) });
|
||||
await expect(pre).toContainText('log/f_89.png');
|
||||
await expect.poll(async () => { const m = await metrics(); return m.top >= m.max - 2; }).toBe(true);
|
||||
// Scrolled up: the position stays while more lines arrive.
|
||||
await pre.evaluate(node => { node.scrollTop = 40; });
|
||||
await step(request, job.id, { processed: 120, records: more(90, 30) });
|
||||
await expect(pre).toContainText('log/f_119.png');
|
||||
expect((await metrics()).top).toBe(40);
|
||||
});
|
||||
|
||||
test('a failed report download says so and leaves the button usable', async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1600, height: 1100 });
|
||||
const job = await startJob(request);
|
||||
await finish(request, job.id);
|
||||
await openJob(page, job.id);
|
||||
await page.route(`**/prompt_manager/jobs/${job.id}/report`, route => route.fulfill({ status: 500, body: 'no' }));
|
||||
const button = heading(page).getByRole('button', { name: 'Download report', exact: true });
|
||||
await button.click();
|
||||
await expect(toast(page)).toHaveText('The report could not be downloaded. Try again.');
|
||||
await expect(button).not.toHaveAttribute('aria-disabled', 'true');
|
||||
});
|
||||
|
||||
@@ -2094,6 +2094,72 @@ class TestActivityJobs(AdminAPITestCase):
|
||||
(legacy["processed"], legacy["found"], legacy["failed"]), (6, 4, 1)
|
||||
)
|
||||
|
||||
async def test_report_is_attachment_without_absolute_paths(self):
|
||||
"""Task 5: the report downloads as JSON, whole and without server paths."""
|
||||
import re
|
||||
|
||||
from py.api.jobs import MAX_OUTCOME_ROWS
|
||||
|
||||
async def runner(job):
|
||||
absolute = f"{self.tmpdir}/output/deep/abs.png"
|
||||
job.record(
|
||||
"added",
|
||||
{"path": absolute, "root": "output", "root_index": 0, "detail": "ok"},
|
||||
f"+ {absolute} · ok",
|
||||
)
|
||||
for index in range(MAX_OUTCOME_ROWS + 5):
|
||||
job.record(
|
||||
"known",
|
||||
{"path": f"k/{index}.png", "root": "output", "detail": "x"},
|
||||
f"= k/{index}.png",
|
||||
)
|
||||
job.update(processed=MAX_OUTCOME_ROWS + 6, total=MAX_OUTCOME_ROWS + 6)
|
||||
|
||||
job = self.api._jobs.start(
|
||||
"scan",
|
||||
"Scan for metadata",
|
||||
record_changing=True,
|
||||
scope={"files": "new", "output_folders": 1},
|
||||
runner=runner,
|
||||
groups=("added", "known", "review", "no_metadata", "errors"),
|
||||
)
|
||||
await asyncio.wait_for(job.task, 10)
|
||||
|
||||
resp = await self.client.request("GET", f"/prompt_manager/jobs/{job.id}/report")
|
||||
text = await resp.text()
|
||||
self.assertEqual(resp.status, 200)
|
||||
self.assertIn("application/json", resp.headers["Content-Type"])
|
||||
disposition = resp.headers["Content-Disposition"]
|
||||
self.assertRegex(
|
||||
disposition,
|
||||
r'^attachment; filename="prompt-manager-scan-\d{8}-\d{4}\.json"$',
|
||||
)
|
||||
self.assertNotIn(self.tmpdir, text)
|
||||
self.assertIsNone(
|
||||
re.search(r"[A-Za-z]:[\\/]|(?<![\w.])/(?:tmp|home|var)/", text)
|
||||
)
|
||||
report = json.loads(text)
|
||||
self.assertEqual(report["job"]["id"], job.id)
|
||||
self.assertEqual(report["job"]["scope"]["files"], "new")
|
||||
self.assertEqual(report["job"]["counters"]["known"], MAX_OUTCOME_ROWS + 5)
|
||||
added = report["outcomes"]["added"]
|
||||
self.assertEqual(
|
||||
(added["count"], len(added["rows"]), added["truncated"]), (1, 1, False)
|
||||
)
|
||||
self.assertEqual(added["rows"][0]["path"], "abs.png")
|
||||
known = report["outcomes"]["known"]
|
||||
self.assertEqual(
|
||||
(known["count"], len(known["rows"]), known["truncated"]),
|
||||
(MAX_OUTCOME_ROWS + 5, MAX_OUTCOME_ROWS, True),
|
||||
)
|
||||
self.assertEqual(report["outcomes"]["errors"]["rows"], [])
|
||||
self.assertFalse(report["outcomes"]["errors"]["truncated"])
|
||||
self.assertEqual(report["log"], job.detail()["log"])
|
||||
self.assertIn("generated_at", report)
|
||||
|
||||
resp = await self.client.request("GET", "/prompt_manager/jobs/nope/report")
|
||||
self.assertEqual(resp.status, 404)
|
||||
|
||||
async def test_cancel_after_first_batch_keeps_partial_counts(self):
|
||||
"""Review M7: the real scan stops at the batch boundary, keeping batch 1."""
|
||||
import threading
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* concept's classes.
|
||||
*
|
||||
* - Header: the page heading names the job; its line is the Activity item's (D54), and its actions are Download report
|
||||
* (disabled until Activity-2) and, while the job runs, "Cancel scan" (danger).
|
||||
* (the job's JSON report, D59) and, while the job runs, "Cancel scan" (danger).
|
||||
* - Progress (`section.surface`): `<progress>`, the server's "Processing file N of M · about X min left", and the
|
||||
* outcome counters as `dl.stat-row` tiles: Added to Library (green), Already known, Needs review (warning), No usable
|
||||
* metadata, Errors (red).
|
||||
@@ -21,6 +21,7 @@ import { showWorkspaceToast } from './toast.mjs';
|
||||
import { noun } from './plural.mjs';
|
||||
import { parseJob, JOBS_URL, LOST_DETAIL } from './activity-store.mjs';
|
||||
import { jobDetail, CANCEL_FAILED_TOAST } from './activity.mjs';
|
||||
import { downloadJobReport } from './activity-scan.mjs';
|
||||
import { createMediaViewer } from './media-viewer.mjs';
|
||||
import { createInspectionController } from './inspection-state.mjs';
|
||||
import { createPromptSender } from './prompt-sender.mjs';
|
||||
@@ -122,7 +123,6 @@ function section(title, id) {
|
||||
export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
const parts = pageHeading(main);
|
||||
const download = createButton('Download report');
|
||||
download.disabled = true;
|
||||
const cancel = createButton('Cancel scan', 'danger');
|
||||
|
||||
const progressPart = section('Progress', 'job-progress-title');
|
||||
@@ -189,6 +189,7 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
let readTimer = null;
|
||||
let roots = null;
|
||||
let lastState = null;
|
||||
let logOpened = false;
|
||||
// Set once this page's own list read answered, or the detail read said the job is unknown.
|
||||
let listedOnce = false;
|
||||
let notFound = false;
|
||||
@@ -290,9 +291,11 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
logLabel.textContent = `● ${scan ? 'Scan' : 'Job'} log`;
|
||||
const text = detail?.log.length ? detail.log.join('\n') : 'No log lines yet.';
|
||||
if (pre.textContent !== text) {
|
||||
// Replacing the text clamps the scroll position, so keep it: pinned readers follow the end, others stay.
|
||||
const { scrollTop } = pre;
|
||||
const atEnd = pre.scrollTop + pre.clientHeight >= pre.scrollHeight - 2;
|
||||
pre.textContent = text;
|
||||
if (atEnd) pre.scrollTop = pre.scrollHeight;
|
||||
pre.scrollTop = atEnd ? pre.scrollHeight : scrollTop;
|
||||
}
|
||||
renderRows(job);
|
||||
}
|
||||
@@ -347,10 +350,21 @@ export function mountActivityJobView(main, { store, scope, api, jobId }) {
|
||||
}
|
||||
|
||||
scope.listen(log, 'toggle', () => {
|
||||
// A log opened for the first time starts at its newest line.
|
||||
if (log.open && !logOpened) { logOpened = true; pre.scrollTop = pre.scrollHeight; }
|
||||
const job = store.find(jobId);
|
||||
if (log.open && job && job.state !== 'lost' && detailChanged(job)) scheduleRead();
|
||||
});
|
||||
|
||||
scope.listen(download, 'click', async () => {
|
||||
if (download.getAttribute('aria-disabled') === 'true') return;
|
||||
download.setAttribute('aria-disabled', 'true');
|
||||
try {
|
||||
await downloadJobReport(jobId);
|
||||
} finally {
|
||||
download.removeAttribute('aria-disabled');
|
||||
}
|
||||
});
|
||||
scope.listen(cancel, 'click', async () => {
|
||||
if (cancel.getAttribute('aria-disabled') === 'true') return;
|
||||
cancel.setAttribute('aria-disabled', 'true');
|
||||
|
||||
@@ -13,6 +13,9 @@ import { navigateTo } from './state.mjs';
|
||||
export const BACKUP_URL = '/prompt_manager/backup';
|
||||
export const BACKUP_DONE = 'Database backup downloaded.';
|
||||
export const BACKUP_FAILED = 'The database could not be backed up. Try again.';
|
||||
export const REPORT_DONE = 'Job report downloaded.';
|
||||
export const REPORT_FAILED = 'The report could not be downloaded. Try again.';
|
||||
const JOBS_REPORT_BASE = '/prompt_manager/jobs';
|
||||
export const START_FAILED = 'The scan could not start. Try again.';
|
||||
const FALLBACK_NAME = 'prompts_backup.db';
|
||||
|
||||
@@ -22,20 +25,25 @@ export function backupFilename(disposition) {
|
||||
return match ? match[1] : FALLBACK_NAME;
|
||||
}
|
||||
|
||||
/** Save a fetched response through a temporary link, named by `filename`. */
|
||||
function saveResponse(blob, filename) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
document.body.append(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
}
|
||||
|
||||
/** Download the prompts database copy the legacy admin page also uses; toasts the outcome. */
|
||||
export async function downloadDatabaseBackup({ fetchUrl = window.ApiClient.fetchUrl, notify = showWorkspaceToast } = {}) {
|
||||
try {
|
||||
const response = await fetchUrl(BACKUP_URL);
|
||||
if (!response.ok) throw new Error(`Backup failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = backupFilename(response.headers.get('Content-Disposition'));
|
||||
document.body.append(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition')));
|
||||
notify(BACKUP_DONE);
|
||||
return true;
|
||||
} catch {
|
||||
@@ -44,6 +52,21 @@ export async function downloadDatabaseBackup({ fetchUrl = window.ApiClient.fetch
|
||||
}
|
||||
}
|
||||
|
||||
/** Download a job's JSON report (Activity-2, D59) through the api-client URL, so a proxy mount works; toasts the outcome. */
|
||||
export async function downloadJobReport(jobId, { fetchUrl = window.ApiClient.fetchUrl, notify = showWorkspaceToast } = {}) {
|
||||
try {
|
||||
const response = await fetchUrl(`${JOBS_REPORT_BASE}/${encodeURIComponent(jobId)}/report`);
|
||||
if (!response.ok) throw new Error(`Report failed with ${response.status}`);
|
||||
const blob = await response.blob();
|
||||
saveResponse(blob, backupFilename(response.headers.get('Content-Disposition')).replace(FALLBACK_NAME, 'prompt-manager-report.json'));
|
||||
notify(REPORT_DONE);
|
||||
return true;
|
||||
} catch {
|
||||
notify(REPORT_FAILED);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** The toast for a refused start, linking the job that holds the slot. */
|
||||
export function busyToast(running) {
|
||||
if (!running) return 'Another job is running. Try again when it ends.';
|
||||
|
||||
Reference in New Issue
Block a user