From 0a568c0f2c8ed6abd76acc0dc6e023040219896d Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Tue, 27 Oct 2026 08:48:17 -0700 Subject: [PATCH] test(system): require a running test and a control for every covered capability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for the legacy coverage proof: - A title declared only with test.skip, .fixme or .fail no longer proves a row (.only and .slow still do). - Every covered row names its native control, except the rows in CONTROL_EXEMPTIONS, each with its reason (today only the gallery per-image auto-tag row, owned by the Images selection bar). The browser test asserts that the covered rows without a control are exactly the exemptions. - JavaScript comments are stripped across the whole source, wherever a block comment starts, skipping strings, template literals and regular expressions. A Python def test_ counts only at function or method indentation and outside comments and docstrings. - A control name several rows share ("Review plan", "Review options", "Refresh") is looked up inside the row the table names (· in ""). - The diagnostics comparison records each select's chosen option, so a wrong default level is caught. Part of #329 --- TESTING.md | 38 ++++-- .../system-diagnostics-concept.spec.mjs | 11 +- tests/browser/workspace-system.spec.mjs | 23 ++-- tests/vendor/legacy-coverage.test.mjs | 54 +++++++- tools/vendor/legacy-coverage.mjs | 127 ++++++++++++++---- 5 files changed, 197 insertions(+), 56 deletions(-) diff --git a/TESTING.md b/TESTING.md index 9e6e4be2..f3030469 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3819,16 +3819,28 @@ S12, #329). Status is `covered` (a native control replaces it), `kept in legacy` on the legacy page, which keeps working) or `approved removal` (a ruling retired it). Native control is `` `
` · · "" ``, `
` a System hash id and `` one of button, combobox, switch, slider, spinbutton, textbox, group, status or -term; it is `—` when the owner is a legacy page, a ruling or another page. Test is the exact -title of a declared test, then its file. Anchors are line numbers on `main` at this PR. +term, followed by ` · in ""` when several rows of the section share the name. +Every `covered` row names one, except the rows in `CONTROL_EXEMPTIONS` +(`tools/vendor/legacy-coverage.mjs`), each with its reason: today only "Gallery per-image +auto-tag / review", which the Images selection bar owns, not a System section. A row that +is not `covered` names none (`—`). Test is the exact title of a test that runs, then its +file. Anchors are line numbers on `main` at this PR. -`tests/vendor/legacy-coverage.test.mjs` (with `tools/vendor/legacy-coverage.mjs`) fails -on a row with no owner, a status outside the three, a malformed Native control cell, a -control on a row that is not `covered`, or a test title its file does not declare (`test(`, -`it(` or `def test_`; a title that appears only in a comment does not count). CI running +`tests/vendor/legacy-coverage.test.mjs` fails on a row with any of these problems: +- no owner, or a status outside the three; +- a malformed Native control cell; +- a covered row without a control (unless exempt), or a control on a row that is not + covered; +- a test title its file does not declare: `test(` or `it(` (`.only` and `.slow` count), + or a `def test_` at function or method indentation; +- a title whose test is only declared with `.skip`, `.fixme` or `.fail`. + +A title that appears only in a comment or a Python docstring does not count. CI running those tests green is what proves them. `every covered legacy System capability has its native control` (workspace-system.spec) opens each covered row's section and finds the -control by role and accessible name. +control by role and accessible name, inside its row when the cell names one. The shared +"Review plan" of the per-operation review row and the Sources "Refresh" of the monitoring +list stay unscoped: any row's control proves them. | Capability | Legacy anchor | Owner | Status | Native control | Test | | --- | --- | --- | --- | --- | --- | @@ -3847,11 +3859,11 @@ control by role and accessible name. | Library export | `py/api/library.py:223` | S9 Backup & export | covered | `backups` · button · "Export JSON" | Export JSON downloads /library/export; secondary buttons download all prompts (/export) and /pairs/export (D103) (workspace-system-backups.spec.mjs) | | Diagnostics run | `admin.html:171,659`; `admin.js:1984` | S12 Diagnostics | covered | `diagnostics` · button · "Run checks" | Run checks opens the shared Health report (workspace-system-diagnostics.spec.mjs) | | Synthetic image-link test | `admin.html:662`; `admin.js:2101` | S12 Diagnostics | covered | `diagnostics` · button · "Create test link…" | Create test link confirms first and names the prompt (workspace-system-diagnostics.spec.mjs) | -| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | `admin.html:692–748`; `admin.js:1834` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) | +| Maintenance cleanup_duplicates, vacuum, cleanup_orphaned_images, prune_orphaned_prompts | `admin.html:692–748`; `admin.js:1834` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | Apply reviewed operation closes, toasts, opens the job in Activity (workspace-system-maintenance.spec.mjs) | | Maintenance check_hash_duplicates, check_consistency | `admin.html:692–748`; `admin.js:1834` | S2 Health report (Maintenance notice, D123) | covered | `maintenance` · button · "Run health checks" | Run health checks opens Health report and updates metric 4 (workspace-system.spec.mjs) | | Maintenance statistics (database info) | `admin.html:692–748`; `admin.js:1834` | S12 Diagnostics | covered | `diagnostics` · term · "Database" | meta-grid shows real database and source status, the pinned Viewer.js version and the host renderers (workspace-system-diagnostics.spec.mjs) | | Maintenance select all / clear all / run | `admin.html:754–762`; `admin.js:1822–1833` | S7 per-operation review | covered | `maintenance` · button · "Review plan" | plan dialog shows the server's scope, protected, backup policy and media lines with the backup box (D100); Cancel writes nothing (workspace-system-maintenance.spec.mjs) | -| Prompt cleanup `/cleanup` | `py/api/admin.py:620` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" | test_duplicate_prompts_plan_counts_the_legacy_groups (test_maintenance_plans.py) | +| Prompt cleanup `/cleanup` | `py/api/admin.py:620` | S6/S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Duplicate prompt text" | test_duplicate_prompts_plan_counts_the_legacy_groups (test_maintenance_plans.py) | | Database backup | `admin.html:179,824`; `admin.js:2139,2454` | S9 Backup & export | covered | `backups` · button · "Back up" | Back up downloads through the api client and revokes the object URL (D103) (workspace-system-backups.spec.mjs) | | Restore from file | `admin.html:183,616–642`; `admin.js:2177–2248` | S8/S9 Backup & export | covered | `backups` · button · "Review restore…" | Review restore stages the chosen file and shows the real summary with the backup box (D105) (workspace-system-backups.spec.mjs) | | Metadata scan with progress, cancel, pill | `admin.html:187,805–865`; `admin.js:2336–2500` | S4 Sources | covered | `sources` · button · "Review scan" | Review scan opens the shared Review metadata scan with all sources (workspace-system-sources.spec.mjs) | @@ -3870,10 +3882,10 @@ control by role and accessible name. | Batch auto-tag output images | `admin.html:191,1210,1279–1318`; `admin.js:3748–3840` | S11 Auto-tagging | covered | `tagging` · button · "Review" | Tag output images reviews the scope and starts a batch job (workspace-system-tagging.spec.mjs) | | Batch review loop | `admin.html:1206,1218–1364`; `admin.js:3841–4172` | S11 + image-tagging plan | covered | `tagging` · button · "Preview tagging review" | Preview tagging review opens the Tag image dialog for the selected image (workspace-system-tagging.spec.mjs) | | Gallery per-image auto-tag / review | `gallery.html:601,605`; `gallery.js:3506,3674–3753` | Image-tagging plan (selection bar) | covered | — | Auto-tag selected with Apply automatically runs a job with the chosen model, thresholds and suggestion count (workspace-tagging-bulk.spec.mjs) | -| Gallery thumbnail generation | `gallery.js:1852,1967` | S5/S7 | covered | `sources` · button · "Review options" | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) | -| Clear server thumbnails | `gallery.js:2175–2192` | S7 Maintenance | covered | `maintenance` · button · "Review options" | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) | -| Browser cache clear | `gallery.js:2194–2207` | S7 Maintenance | covered | `maintenance` · button · "Review plan" | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) | -| Refresh file listing | `gallery.js:2209` | S4 Sources | covered | `sources` · button · "Refresh" | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) | +| Gallery thumbnail generation | `gallery.js:1852,1967` | S5/S7 | covered | `sources` · button · "Review options" · in "Thumbnail generation" | Start closes the dialog, toasts, and opens the job in Activity (workspace-system-sources.spec.mjs) | +| Clear server thumbnails | `gallery.js:2175–2192` | S7 Maintenance | covered | `maintenance` · button · "Review options" · in "Thumbnail cache" | Thumbnail cache opens the S5 dialog (workspace-system-maintenance.spec.mjs) | +| Browser cache clear | `gallery.js:2194–2207` | S7 Maintenance | covered | `maintenance` · button · "Review plan" · in "Metadata cache" | Metadata cache reviews and clears only owned browser entries when there are some (D102) (workspace-system-maintenance.spec.mjs) | +| Refresh file listing | `gallery.js:2209` | S4 Sources | covered | `sources` · button · "Refresh" · in "Refresh files" | Refresh files calls /gallery/rescan and toasts its counts (workspace-system-sources.spec.mjs) | | Duplicate image scan, review, delete | `gallery.js:2234–2577` | S6/S7 Maintenance | covered | `maintenance` · button · "Find duplicates" | Find duplicates starts a job; its view links Review groups; Keep radios build a duplicate_files plan (D101) (workspace-system-maintenance.spec.mjs) | | Default view mode, image quality | `gallery-settings.js:26–27` | S3 Appearance | covered | `appearance` · combobox · "Default layout" | defaults apply to a new visit but not to the current view or a saved view (workspace-system.spec.mjs) | | Auto-load metadata, cache metadata, show file paths | `gallery-settings.js:31–33` | S3 Appearance | covered | `appearance` · switch · "Load metadata on selection" | metadata off shows an explicit load and loads once on request (workspace-system.spec.mjs) | diff --git a/tests/browser/system-diagnostics-concept.spec.mjs b/tests/browser/system-diagnostics-concept.spec.mjs index 94e296d4..af459cd3 100644 --- a/tests/browser/system-diagnostics-concept.spec.mjs +++ b/tests/browser/system-diagnostics-concept.spec.mjs @@ -58,6 +58,8 @@ function measureDiagnostics([mainSelector, lineFormat]) { const toolbarItem = node => ({ name: name(node), text: node.localName === 'select' || node.localName === 'input' ? null : text(node), for: node.localName === 'label' ? node.htmlFor : null, id: node.id || null, options: node.localName === 'select' ? [...node.options].map(option => option.textContent) : null, + // The chosen option, so a wrong default level is noticed. + selected: node.localName === 'select' ? node.selectedOptions[0]?.textContent ?? null : null, input: node.localName === 'input' ? [node.type, node.min, node.max, node.value] : null, className: node.localName === 'button' ? node.className : null }); const surfaces = [...layout.querySelectorAll(':scope > section.surface')]; @@ -190,9 +192,10 @@ function mapDiagnosticsConcept(reference, { report, files, stats }) { // D109: all six minimum levels (the concept has three). { ...level, options: ['All levels', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] }, // D109: the added Lines control, 1–500, default 100. - { name: 'label.small', text: 'Lines', for: 'log-limit', id: null, options: null, input: null, className: null }, - { name: 'input', text: null, for: null, id: 'log-limit', options: null, input: ['number', '1', '500', '100'], + { name: 'label.small', text: 'Lines', for: 'log-limit', id: null, options: null, selected: null, input: null, className: null }, + { name: 'input', text: null, for: null, id: 'log-limit', options: null, selected: null, + input: ['number', '1', '500', '100'], className: null }, refresh, // D109: the active log, not a sample. { ...download, text: 'Download log' }, @@ -358,6 +361,10 @@ test('diagnostics comparison detects mutations', async ({ page, request }) => { await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.append(select.options[1]); }); expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.1.options.1'])); await page.evaluate(() => { const select = document.querySelector('#workspace-main #log-level'); select.insertBefore(select.options[5], select.options[1]); }); + // A wrong default level. + await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = 'INFO'; }); + expect(await paths()).toEqual(['surfaces.1.toolbar.1.selected']); + await page.evaluate(() => { document.querySelector('#workspace-main #log-level').value = ''; }); // A primary Refresh. await page.evaluate(() => { document.querySelectorAll('#workspace-main .toolbar > button')[0].classList.add('ca-primary'); }); expect(await paths()).toEqual(expect.arrayContaining(['surfaces.1.toolbar.4.className'])); diff --git a/tests/browser/workspace-system.spec.mjs b/tests/browser/workspace-system.spec.mjs index 51c39057..7dab8663 100644 --- a/tests/browser/workspace-system.spec.mjs +++ b/tests/browser/workspace-system.spec.mjs @@ -1,7 +1,7 @@ import { startCoverage, saveCoverage } from '../../tools/vendor/browser-coverage.mjs'; import { test, expect } from './fixtures.mjs'; import { seedConcept } from './concept-inventory.mjs'; -import { readCoverageTable, parseControl } from '../../tools/vendor/legacy-coverage.mjs'; +import { readCoverageTable, parseControl, CONTROL_EXEMPTIONS } from '../../tools/vendor/legacy-coverage.mjs'; // System › Overview (#249 System S2 Task 6) on the concept-shaped real fixture: the metrics, rows, Statistics dialog and // Health report read the server, never sample values (D93, D95, D96). The hooks are workspace_system_fixture.py's. @@ -812,8 +812,11 @@ test('every covered legacy System capability has its native control', async ({ p }); async function findControls(page) { - const controls = readCoverageTable().filter(row => row.Status === 'covered') - .map(row => ({ capability: row.Capability, ...parseControl(row['Native control']) })).filter(row => row.section); + const covered = readCoverageTable().filter(row => row.Status === 'covered') + .map(row => ({ capability: row.Capability, ...parseControl(row['Native control']) })); + // A covered row without a control is one of the recorded exemptions, never silently skipped. + expect(covered.filter(row => !row.section).map(row => row.capability)).toEqual(Object.keys(CONTROL_EXEMPTIONS)); + const controls = covered.filter(row => row.section); expect(controls.length).toBeGreaterThan(40); const sections = [...new Set(controls.map(row => row.section))]; for (const section of sections) { @@ -821,14 +824,16 @@ async function findControls(page) { const main = page.locator('#workspace-main'); await expect(layoutOf(page).locator(':scope > *').first()).toBeVisible(); for (const details of await main.locator('details:not([open]) > summary').all()) await details.click(); - for (const { capability, role, name } of controls.filter(row => row.section === section)) { + for (const { capability, role, name, row } of controls.filter(item => item.section === section)) { + // A name several rows share is looked up inside the row the table names. + const within = row ? main.locator('.operation').filter({ has: page.locator(`h3:text-is("${row}")`) }) : main; const exact = new RegExp(`^${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`); // A `dt` (term) and a `details` disclosure (group) have no name from content: the term's text, the summary's. - const control = role === 'term' ? main.getByRole('term').filter({ hasText: exact }) - : role === 'group' ? main.locator('details').filter({ has: page.locator(':scope > summary', { hasText: exact }) }) - .or(main.getByRole('group', { name, exact: true })) - : main.getByRole(role, { name, exact: true }); - await expect(control.first(), `${capability}: ${section} · ${role} · "${name}"`).toBeVisible(); + const control = role === 'term' ? within.getByRole('term').filter({ hasText: exact }) + : role === 'group' ? within.locator('details').filter({ has: page.locator(':scope > summary', { hasText: exact }) }) + .or(within.getByRole('group', { name, exact: true })) + : within.getByRole(role, { name, exact: true }); + await expect(control.first(), `${capability}: ${section} · ${role} · "${name}"${row ? ` in "${row}"` : ''}`).toBeVisible(); } } } diff --git a/tests/vendor/legacy-coverage.test.mjs b/tests/vendor/legacy-coverage.test.mjs index f1aafb07..c1bf38d7 100644 --- a/tests/vendor/legacy-coverage.test.mjs +++ b/tests/vendor/legacy-coverage.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { COLUMNS, STATUSES, declaredTitles, parseControl, parseCoverageTable, readCoverageTable, rowProblems } - from '../../tools/vendor/legacy-coverage.mjs'; +import { COLUMNS, CONTROL_EXEMPTIONS, STATUSES, declaredTitles, parseControl, parseCoverageTable, readCoverageTable, + rowProblems } from '../../tools/vendor/legacy-coverage.mjs'; // TESTING.md "Legacy System capability coverage" (#249 System S12, #329): every legacy System capability names an // owner, a known status, a well-formed native control and a test its file really declares. @@ -42,17 +42,29 @@ test('a title that appears only in a comment does not count', () => { const declared = `${commented}\nfor (const theme of THEMES) {\n test('clears the logs', async ({ page }) => {});\n}\n`; assert.deepEqual(rowProblems(ROW, declaring(declared)), []); // Escaped quotes, template literals, modifiers and a title on the next line are declarations too. - const titles = declaredTitles("test('a gate\\'s refusal', () => {});\ntest.skip(`b`, () => {});\nit(\n 'c', () => {});", + const titles = declaredTitles("test('a gate\\'s refusal', () => {});\ntest.only(`b`, () => {});\nit(\n 'c', () => {});", 'x.test.js'); assert.deepEqual([...titles], ["a gate's refusal", 'b', 'c']); + // A block comment that starts after code on a line, and a `//` or `/*` inside a string or a regular expression. + const trailing = [ + "const url = 'http://host/*'; /* test('clears the logs', () => {});", + "test('clears the logs', () => {}); */", + "const pattern = /['\"`]\\/\\*/;", + "test('after the pattern', () => {});", + ].join('\n'); + assert.deepEqual([...declaredTitles(trailing, 'x.spec.mjs')], ['after the pattern']); // Python: only a real `def test_…` counts, never a commented one. - const python = '# def test_report_lists_config_only_keys(self):\nclass T:\n def test_other(self):\n pass\n'; - assert.deepEqual([...declaredTitles(python, 'test_x.py')], ['test_other']); + const python = ['# def test_report_lists_config_only_keys(self):', 'class T:', ' \"\"\"Example:', '', + ' def test_in_a_docstring(self):', ' \"\"\"', ' def test_other(self):', ' def test_nested():', + ' pass', 'def test_module():', ' pass', ''].join('\n'); + assert.deepEqual([...declaredTitles(python, 'test_x.py')], ['test_other', 'test_module']); }); test('a malformed native control cell fails', () => { assert.deepEqual(parseControl('`diagnostics` · button · "Clear all logs…"'), - { section: 'diagnostics', role: 'button', name: 'Clear all logs…' }); + { section: 'diagnostics', role: 'button', name: 'Clear all logs…', row: null }); + assert.deepEqual(parseControl('`maintenance` · button · "Review plan" · in "Duplicate prompt text"'), + { section: 'maintenance', role: 'button', name: 'Review plan', row: 'Duplicate prompt text' }); assert.equal(parseControl('—'), null); for (const [cell, message] of [ ['diagnostics · button · "Clear all logs…"', /malformed/], @@ -72,3 +84,33 @@ test('a malformed native control cell fails', () => { assert.throws(() => parseCoverageTable(`### Legacy System capability coverage\n\n| Capability | Owner |\n| --- | --- |\n`), /columns/); assert.deepEqual(COLUMNS, ['Capability', 'Legacy anchor', 'Owner', 'Status', 'Native control', 'Test']); }); + +test('a skipped test does not prove a capability', () => { + for (const modifier of ['skip', 'fixme', 'fail']) { + assert.deepEqual(rowProblems(ROW, declaring(`test.${modifier}('clears the logs', async () => {});`)), + ['clear.spec.mjs only skips "clears the logs"'], modifier); + } + for (const modifier of ['only', 'slow']) { + assert.deepEqual(rowProblems(ROW, declaring(`test.${modifier}('clears the logs', async () => {});`)), [], modifier); + } + // A running declaration beside a skipped one counts. + assert.deepEqual(rowProblems(ROW, declaring("test.skip('clears the logs', () => {});\ntest('clears the logs', () => {});")), []); +}); + +test('a covered row names its native control unless it is exempt', () => { + const source = declaring("test('clears the logs', () => {});"); + assert.deepEqual(rowProblems({ ...ROW, 'Native control': '—' }, source), ['a covered row names no native control']); + for (const status of ['kept in legacy', 'approved removal']) { + assert.deepEqual(rowProblems({ ...ROW, Status: status, 'Native control': '—' }, source), [], status); + } + const [exempt] = Object.keys(CONTROL_EXEMPTIONS); + assert.deepEqual(rowProblems({ ...ROW, Capability: exempt, 'Native control': '—' }, source), []); + // Every exemption has a reason and names a covered row of the table that has no control. + const rows = readCoverageTable(); + for (const [capability, reason] of Object.entries(CONTROL_EXEMPTIONS)) { + assert.ok(reason.length > 20, capability); + const row = rows.find(item => item.Capability === capability); + assert.ok(row, `${capability} is in the table`); + assert.deepEqual([row.Status, row['Native control']], ['covered', '—']); + } +}); diff --git a/tools/vendor/legacy-coverage.mjs b/tools/vendor/legacy-coverage.mjs index 7de3dd38..46a807c7 100644 --- a/tools/vendor/legacy-coverage.mjs +++ b/tools/vendor/legacy-coverage.mjs @@ -16,6 +16,13 @@ export const ROLES = Object.freeze(['button', 'combobox', 'switch', 'slider', 's // Where a Test cell's file is looked up, in order. export const TEST_FOLDERS = Object.freeze(['tests/browser', 'tests/js', 'tests/vendor', 'tests']); const NONE = '—'; +/** + * Covered rows whose replacement is not on a System section, so they name no native control (`—`), each with why. Every + * other covered row must name one. + */ +export const CONTROL_EXEMPTIONS = Object.freeze({ + 'Gallery per-image auto-tag / review': 'owned by the Images selection bar (Auto-tag selected), not a System section', +}); /** Split a Markdown table row into its cells (a `|` inside backticks is not a separator). */ function cells(line) { @@ -54,15 +61,18 @@ export function parseCoverageTable(markdown) { return rows; } -/** A Native control cell: `` `
` · · "" `` → {section, role, name}; `—` → null; else it throws. */ +/** + * A Native control cell: `` `
` · · "" ``, optionally ` · in ""` when the name is shared + * by several rows of the section → {section, role, name, row}; `—` → null; else it throws. + */ export function parseControl(cell) { if (cell === NONE) return null; - const match = /^`([a-z]+)` · ([a-z]+) · "([^"]+)"$/.exec(cell); + const match = /^`([a-z]+)` · ([a-z]+) · "([^"]+)"(?: · in "([^"]+)")?$/.exec(cell); if (!match) throw new Error(`malformed native control "${cell}"`); - const [, section, role, name] = match; + const [, section, role, name, row = null] = match; if (!SECTIONS.includes(section)) throw new Error(`unknown System section "${section}"`); if (!ROLES.includes(role)) throw new Error(`unknown role "${role}"`); - return { section, role, name }; + return { section, role, name, row }; } /** A Test cell: ` (<file>)` → {title, file}. */ @@ -72,30 +82,88 @@ export function parseTest(cell) { return { title: match[1], file: match[2] }; } -/** - * The titles a source declares: a line that starts with `test('…'` or `it('…'` (with `.only`, `.skip`, `.fixme`, - * `.fail` or `.slow`) in JavaScript, `def test_…` in Python. Comments never count: a JavaScript line inside a block - * comment or starting with `//` or `*`, and a Python line starting with `#`. - */ -export function declaredTitles(source, file) { - if (file.endsWith('.py')) { - return new Set([...source.matchAll(/^[ \t]*(?:async[ \t]+)?def[ \t]+(test_\w+)[ \t]*\(/gm)].map(match => match[1])); +// Modifiers that still run the test; `.skip`, `.fixme` and `.fail` do not prove the capability. +const RUNS = new Set(['', 'only', 'slow']); +// After these characters (or at the start) a `/` opens a regular expression, not a division. +const REGEX_BEFORE = /[(,=:[!&|?{};+\-*%<>~^]$/; + +/** Where the string, template literal or regular expression opened by `source[start]` ends (its closing index), or -1. */ +function literalEnd(source, start) { + const quote = source[start]; + let inClass = false; + for (let end = start + 1; end < source.length; end += 1) { + const at = source[end]; + if (at === '\\') end += 1; + else if (quote === '/' && at === '[') inClass = true; + else if (quote === '/' && at === ']') inClass = false; + else if (at === quote && !inClass) return end; + // A quoted string or a regular expression never spans lines. + else if (at === '\n' && quote !== '`') return -1; } - let comment = false; - const code = source.split('\n').map(line => { - const text = line.trim(); - if (comment) { - if (text.includes('*/')) comment = false; - return ''; + return -1; +} + +/** + * JavaScript `source` with every comment blanked (newlines kept), wherever it starts on a line. Strings, template + * literals and regular expressions are skipped whole, so a `//` or `/*` inside them is not a comment. + */ +export function stripJsComments(source) { + let out = ''; + let index = 0; + while (index < source.length) { + const char = source[index]; + const next = source[index + 1]; + if (char === '/' && next === '/') { + while (index < source.length && source[index] !== '\n') index += 1; + continue; } - if (text.startsWith('/*')) { - comment = !text.includes('*/'); - return ''; + if (char === '/' && next === '*') { + const close = source.indexOf('*/', index + 2); + const stop = close === -1 ? source.length : close + 2; + out += source.slice(index, stop).replace(/[^\n]/g, ' '); + index = stop; + continue; } - return text.startsWith('//') || text.startsWith('*') ? '' : line; - }).join('\n'); - const declaration = /^[ \t]*(?:test|it)(?:\.(?:only|skip|fixme|fail|slow))?\(\s*(['"`])((?:\\.|(?!\1)[^\\])*)\1/gm; - return new Set([...code.matchAll(declaration)].map(match => match[2].replace(/\\(.)/g, '$1'))); + const opens = char === "'" || char === '"' || char === '`' || (char === '/' && REGEX_BEFORE.test(out.trimEnd() || '(')); + const end = opens ? literalEnd(source, index) : -1; + if (end === -1) { + out += char; + index += 1; + } else { + out += source.slice(index, end + 1); + index = end + 1; + } + } + return out; +} + +/** Python `source` with its comments and triple-quoted strings (docstrings) blanked, newlines kept. */ +export function stripPythonComments(source) { + return source.replace(/("""|''')[\s\S]*?\1|#[^\n]*/g, text => text.replace(/[^\n]/g, ' ')); +} + +/** + * The tests a source declares, title → whether it runs: a `test('…'` or `it('…'` call at a line's start (`.only` and + * `.slow` run; `.skip`, `.fixme` and `.fail` do not) in JavaScript, a `def test_…` at function or method indentation in + * Python. Comments, and Python docstrings, never count. + */ +export function declarations(source, file) { + const found = new Map(); + const add = (title, runs) => found.set(title, found.get(title) || runs); + if (file.endsWith('.py')) { + for (const match of stripPythonComments(source).matchAll(/^(?: {4})?(?:async[ \t]+)?def[ \t]+(test_\w+)[ \t]*\(/gm)) { + add(match[1], true); + } + return found; + } + const declaration = /^[ \t]*(?:test|it)(?:\.(only|skip|fixme|fail|slow))?\(\s*(['"`])((?:\\.|(?!\2)[^\\])*)\2/gm; + for (const match of stripJsComments(source).matchAll(declaration)) add(match[3].replace(/\\(.)/g, '$1'), RUNS.has(match[1] ?? '')); + return found; +} + +/** The titles of the tests a source declares and runs. */ +export function declaredTitles(source, file) { + return new Set([...declarations(source, file)].filter(([, runs]) => runs).map(([title]) => title)); } /** The repository path of a Test cell's file, or null. */ @@ -121,6 +189,9 @@ export function rowProblems(row, { root = ROOT, locate = file => testFile(file, problems.push(error.message); } if (control && row.Status !== 'covered') problems.push(`a ${row.Status} row names a native control`); + if (row.Status === 'covered' && row['Native control'] === NONE && !Object.hasOwn(CONTROL_EXEMPTIONS, row.Capability)) { + problems.push('a covered row names no native control'); + } let test; try { test = parseTest(row.Test); @@ -130,7 +201,11 @@ export function rowProblems(row, { root = ROOT, locate = file => testFile(file, } const path = locate(test.file); if (!path) problems.push(`no test file ${test.file}`); - else if (!declaredTitles(read(path), test.file).has(test.title)) problems.push(`${test.file} declares no test "${test.title}"`); + else { + const runs = declarations(read(path), test.file).get(test.title); + if (runs === undefined) problems.push(`${test.file} declares no test "${test.title}"`); + else if (!runs) problems.push(`${test.file} only skips "${test.title}"`); + } return problems; }