Merge remote-tracking branch 'origin/main' into fix/413-vocab-restore-flake

A job-list read the page sent before the restore could still be answered
while it ran. The gate now routes every job-list read from before the
first navigation and, before a restore, waits for the reads in flight to
be answered. The history and library editor conversions are pre-emptive:
same race, never seen failing.

tests/browser/unseen-restore.mjs

@@ -0,0 +1,54 @@
+// A restore the open workspace page has not noticed yet (#413).
+//
+// Specs that restore through the API while a page is open check what the page does with a restore it did not see:
+// its next write is refused as "Prompt storage was restored since this dialog opened". But the page's Activity store
+// reads the job list every 15 s (activity-store.mjs POLL_MS), and a read the server answers while the restore job runs
+// makes the page follow it: when it finishes, the page is shown again from the restored data (S9, index.mjs
+// onRestoreFinished), which closes any open dialog. On a slow runner a restore can take longer than the time left to
+// the next read (CI run 40646112668: the restore took 16 s), so whether the page saw it depended on timing.
+//
+// `armUnseenRestores(page)`, before the page loads, passes every job-list read of the page through one route, so it
+// knows which reads the server has not answered yet. `unseenRestore(page, restoring)` then closes the gate, waits
+// for those reads' answers (sent before the restore began, they cannot list it), runs `restoring()`, and only after
+// it has answered lets the held reads go on: they read the restore as already ended, as a tab that read its job list
+// just before and just after the restore does. A restore that is never seen running never reloads the page.
+const JOBS = '**/prompt_manager/jobs';
+const gates = new WeakMap();
+
+/** Route `page`'s job-list reads through the restore gate. Call before the page's first navigation. */
+export async function armUnseenRestores(page) {
+    const gate = { closed: null, inFlight: 0, idle: [] };
+    gates.set(page, gate);
+    await page.route(JOBS, async route => {
+        if (gate.closed) {
+            await gate.closed;
+            return route.fallback();
+        }
+        gate.inFlight += 1;
+        try {
+            const response = await route.fetch();
+            await route.fulfill({ response }).catch(() => {});
+        } catch {
+            await route.abort().catch(() => {});
+        } finally {
+            gate.inFlight -= 1;
+            if (gate.inFlight === 0) for (const resolve of gate.idle.splice(0)) resolve();
+        }
+    });
+}
+
+/** Run `restoring()` (an API restore) unseen by `page`'s Activity store; resolves with what it resolves with. */
+export async function unseenRestore(page, restoring) {
+    const gate = gates.get(page);
+    if (!gate) throw new Error('armUnseenRestores(page) must run before the page loads');
+    let open;
+    gate.closed = new Promise(resolve => { open = resolve; });
+    try {
+        // Reads already on their way were sent before the restore: their answers cannot list it.
+        if (gate.inFlight > 0) await new Promise(resolve => gate.idle.push(resolve));
+        return await restoring();
+    } finally {
+        gate.closed = null;
+        open();
+    }
+}

tests/browser/workspace-history.spec.mjs

@@ -1,5 +1,6 @@
 import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
 import {test,expect} from './fixtures.mjs';
+import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
 import {readFile,writeFile} from 'node:fs/promises';
 import {createHash} from 'node:crypto';
 
@@ -16,13 +17,13 @@ const marker=(page,id,anchor='10')=>recorded(page).locator(`button[data-run-id="
 const exact=async locator=>locator.evaluate(node=>node.textContent);
 
 test.beforeEach(async({page,request,browserName})=>{
-    await startCoverage(page,browserName);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message));
+    await startCoverage(page,browserName);await armUnseenRestores(page);page.historyErrors=[];page.on('pageerror',error=>page.historyErrors.push(error.message));
     const reset=await request.post('/__test__/reset',{data:{chronological_history:true}});await expect(reset).toBeOK();page.fixture=await reset.json();
 });
 test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.historyErrors).toEqual([]);});
 
 const command=async(request,data)=>{const response=await request.post('/__test__/history/commands',{data});await expect(response).toBeOK();return response.json();};
-const restoreSnapshot=async(request,bytes)=>{const response=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}});await expect(response).toBeOK();expect((await response.json()).success).toBe(true);};
+const restoreSnapshot=async(page,request,bytes)=>{const response=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'history.db',mimeType:'application/octet-stream',buffer:bytes}}}));await expect(response).toBeOK();expect((await response.json()).success).toBe(true);};
 const historyRead=async(request,id)=>{const response=await request.get(`/prompt_manager/library/${id}/history`);await expect(response).toBeOK();return response.json();};
 const frozenContexts=contexts=>contexts.map(({navigation,positive_prompt_id,negative_prompt_id,pair_id,...frozen})=>frozen);
 const immutableReceipts=items=>items.map(({current_candidate,...receipt})=>receipt);
@@ -62,7 +63,7 @@ for(const boundary of ['window','detail','candidate','inspection','media']) test
     await command(request,{action:'arm',boundary});const pending=request.get(target);
     try {
         await expect.poll(async()=> (await(await request.get('/__test__/history/checkpoint')).json()).entered).toBe(true);
-        await restoreSnapshot(request,bytes);
+        await restoreSnapshot(page,request,bytes);
     } finally {await command(request,{action:'release'});}
     const retired=await pending;expect(retired.status()).toBe(409);expect((await retired.json()).code).toBe('history-changed');
     for(const url of [`/prompt_manager/library/${f.detail_id}/history?limit=1&cursor=${encodeURIComponent(firstWindow.next_cursor)}`,`${base}?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}`,`${base}/outputs?epoch=${epoch}&cursor=${encodeURIComponent(outputs.next_cursor)}`,media.url,media.thumbnail_url,media.inspection_url]) {
@@ -130,7 +131,7 @@ test('archive delete merge and prune preserve frozen evidence without repairing
 
 test('actual restore with reused numeric owner rejects stale history and offers explicit saved recovery',async({page,request})=>{
     const f=page.fixture;await open(page,f.detail_id);const original=await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'));
-    const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(request,await source.body());
+    const source=await request.get('/__test__/history/reused-snapshot');await expect(source).toBeOK();await restoreSnapshot(page,request,await source.body());
     await marker(page,f.runs.shared).click();await expect(dialog(page)).toContainText('Recorded evidence changed');await page.keyboard.press('Escape');
     await expect(page.locator('.library-parent-status')).toContainText('changed');expect(await exact(saved(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(original);
     await expect(recorded(page).getByRole('button',{name:'Reload recorded history',exact:true})).toBeVisible();
@@ -142,12 +143,12 @@ for(const outcome of ['success','error'])test(`delayed actual old browser ${outc
     const f=page.fixture;await open(page,f.detail_id);const snapshot=await request.get('/__test__/history/snapshot');const bytes=await snapshot.body();
     let entered,release;const started=new Promise(resolve=>{entered=resolve;}),held=new Promise(resolve=>{release=resolve;});
     await page.route(new RegExp(`/history/runs/${f.runs.shared}\\?`),async route=>{
-        if(outcome==='error')await restoreSnapshot(request,bytes);
+        if(outcome==='error')await restoreSnapshot(page,request,bytes);
         const response=await route.fetch();expect(response.status()).toBe(outcome==='success'?200:409);entered();await held;
         await route.fulfill({response}).catch(()=>{});
     });
     try {
-        await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(request,bytes);
+        await marker(page,f.runs.shared).click();await started;if(outcome==='success')await restoreSnapshot(page,request,bytes);
         await page.evaluate(()=>{location.hash='#/overview';});await expect(page.locator('#workspace-main h1')).toHaveText('Overview');release();
         await page.unroute(new RegExp(`/history/runs/${f.runs.shared}\\?`));await open(page,f.detail_id);
         await expect(recorded(page).locator('button[data-run-id]')).toHaveCount(0);
@@ -258,7 +259,7 @@ test('inspection first discovers a real restore and immediately retires every su
                 if(changed)window.recordedRecoveryStates.push({receipts:run.querySelectorAll('[data-receipt]').length,active:[...run.querySelectorAll('[data-send-role],[data-copy-role],[data-view-receipt]')].filter(node=>!node.disabled&&node.checkVisibility()).length});
             });observer.observe(run,{subtree:true,childList:true,attributes:true,characterData:true});
         });
-        await restoreSnapshot(request,await snapshot.body());
+        await restoreSnapshot(page,request,await snapshot.body());
     } finally {await command(request,{action:'release'});}
     const response=await inspectionRead;expect(response.status()).toBe(409);expect((await response.json()).code).toBe('history-changed');
     await expect(dialog(page).locator('.recorded-run-status')).toContainText('Recorded evidence changed');

tests/browser/workspace-library-editor.spec.mjs

@@ -1,6 +1,7 @@
 import {writeFile} from 'node:fs/promises';
 import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
 import {test,expect} from './fixtures.mjs';
+import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
 import {captureCanonicalState} from './history-canonical-comparison.mjs';
 import {inventoryOf,mapConcept,diffInventory} from './library-editor-canonical-comparison.mjs';
 
@@ -13,7 +14,7 @@ const RATINGS=[['5','5 / 5'],['4','4 / 5'],['3','3 / 5'],['2','2 / 5'],['1','1 /
 const editorDialog=page=>page.locator('dialog.library-editor');
 
 test.beforeEach(async({page,request,browserName})=>{
-    await startCoverage(page,browserName);
+    await startCoverage(page,browserName);await armUnseenRestores(page);
     const reset=await request.post('/__test__/library-editor/reset',{data:{}});await expect(reset).toBeOK();page.fixture=await reset.json();
     page.editorErrors=[];page.on('pageerror',error=>page.editorErrors.push(error.message));
     page.writes=[];page.on('request',item=>{if(item.method()!=='GET')page.writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});
@@ -260,7 +261,8 @@ test('recovery states render inline and never replay a write',async({page,reques
     page.writes.length=0;
     dialog=await openEditor(page,{mode:'new'});await ready(dialog);
     const snapshot=await request.get('/__test__/library-editor/snapshot');await expect(snapshot).toBeOK();
-    const restore=await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:await snapshot.body()}}});
+    const buffer=await snapshot.body();
+    const restore=await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer}}}));
     await expect(restore).toBeOK();page.writes.length=0;
     await field(dialog,'Title').fill('After restore');await field(dialog,'Positive prompt').fill('Restored storage prompt');await button(dialog,'Save').click();
     await expect(note(dialog)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was saved.');
@@ -588,7 +590,7 @@ test('saved refresh failure retries only reads',async({page})=>{
 
 test('restored storage after a confirmed save or before an archive offers an explicit Library reload',async({page,request})=>{
     const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body();
-    const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}});
+    const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}));
     await openHistory(page,f.positive_id);
     await page.route(`**/prompt_manager/library/${f.positive_id}`,async route=>{
         if(route.request().method()!=='PATCH')return route.continue();
@@ -615,7 +617,7 @@ test('restored storage after a confirmed save or before an archive offers an exp
 const DISCARDS='Leaving this dialog discards your entries.';
 test('restored storage before an Edit or Add negative save offers only the Library reload exit',async({page,request})=>{
     const f=page.fixture;const snapshot=await(await request.get('/__test__/library-editor/snapshot')).body();
-    const restore=()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}});
+    const restore=()=>unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'editor.db',mimeType:'application/octet-stream',buffer:snapshot}}}));
     await openHistory(page,f.positive_id);
     await page.getByRole('button',{name:'Edit metadata',exact:true}).click();let dialog=editorDialog(page);await ready(dialog);
     await expect(await restore()).toBeOK();

tests/browser/workspace-vocabulary.spec.mjs

@@ -1,6 +1,7 @@
 import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs';
 import {writeFile} from 'node:fs/promises';
 import {test,expect} from './fixtures.mjs';
+import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs';
 import {captureCanonicalState} from './history-canonical-comparison.mjs';
 import {pageInventoryOf,inventoryOf,diffInventory,mapVocabularyConcept,mapVocabularyDialog} from './vocabulary-canonical-comparison.mjs';
 
@@ -40,7 +41,7 @@ async function expectLibrary(page,{tags='',category='',count}) {
 
 let manifest;
 test.beforeEach(async({page,request,browserName})=>{
-    await startCoverage(page,browserName);manifest=await seed(request);
+    await startCoverage(page,browserName);await armUnseenRestores(page);manifest=await seed(request);
     page.vocabularyErrors=[];page.on('pageerror',error=>page.vocabularyErrors.push(error.message));
 });
 test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.vocabularyErrors).toEqual([]);});
@@ -264,7 +265,8 @@ const toast=page=>page.locator('#workspace-toast');
 const actionNames=scope=>scope.locator('.dialog-actions button').evaluateAll(nodes=>nodes.map(node=>node.textContent.trim()));
 const snapshot=async request=>{const response=await request.get('/__test__/vocabulary/snapshot');await expect(response).toBeOK();return response.json();};
 const backup=async request=>{const response=await request.get('/__test__/vocabulary/backup');await expect(response).toBeOK();return response.body();};
-const restore=async(request,buffer)=>expect(await request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}})).toBeOK();
+// A restore the open page has not seen, so its dialog stays to report it (#413).
+const restore=async(page,request,buffer)=>expect(await unseenRestore(page,()=>request.post('/prompt_manager/restore',{multipart:{database_file:{name:'vocabulary.db',mimeType:'application/octet-stream',buffer}}}))).toBeOK();
 const tagNames=rows=>rows.tags.map(([,name])=>name);
 function trackWrites(page) {
     const writes=[];page.on('request',item=>{if(item.method()!=='GET')writes.push({method:item.method(),url:new URL(item.url()).pathname,body:item.postDataJSON()});});return writes;
@@ -482,7 +484,7 @@ test('conflict and restore states keep the dialog and write nothing',async({page
 
     const before=await snapshot(request);
     box=await openDialog(page,main(page).getByRole('button',{name:'+ Create tag',exact:true}),'Create tag');
-    await restore(request,saved);
+    await restore(page,request,saved);
     await field(box,'Tag name').fill('sunrise');await button(box,'Apply').click();
     await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.');
     await expect(button(box,'Apply')).toBeDisabled();
@@ -622,7 +624,7 @@ test('delete confirmation read failure, missing tag and restore keep Delete disa
     await expect(button(box,'Delete')).toBeDisabled();await expect(tagRow(page,'calm').locator('td:nth-child(2)')).toHaveText('2');
     await button(box,'Cancel').click();
 
-    box=await openRename(page,'mist');await restore(request,saved);
+    box=await openRename(page,'mist');await restore(page,request,saved);
     await button(box,'Delete tag…').click();
     await expect(note(box)).toHaveText('Prompt storage was restored since this dialog opened. Nothing was changed.');
     await expect(button(box,'Delete')).toBeDisabled();expect(await actionNames(box)).toEqual(['Reload','Cancel','Delete']);