389 lines
15 KiB
JavaScript
389 lines
15 KiB
JavaScript
/**
|
|
* E2E Tests for Error Boundaries (R5)
|
|
* ====================================
|
|
* Tests the ErrorBoundary component, global error handlers, and privacy sanitization.
|
|
*
|
|
* Test Scenarios:
|
|
* 1. Chat island error shows ErrorBoundary UI
|
|
* 2. Reload button remounts component
|
|
* 3. Copy error ID to clipboard
|
|
* 4. Permanent error after 3 attempts
|
|
* 5. Statistics island isolation (error in one doesn't affect other)
|
|
* 6. Privacy mode sanitizes PII
|
|
*/
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
import { waitForDoctorReady, clearStorage } from '../utils/helpers.js';
|
|
|
|
test.describe('Error Boundaries', () => {
|
|
test.beforeEach(async ({ page }) => {
|
|
// Increase timeout for initialization
|
|
test.setTimeout(45000);
|
|
|
|
// Mock the ComfyUI modules that doctor.js tries to import
|
|
await page.route('**/scripts/app.js', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/javascript',
|
|
body: 'export const app = window.app;',
|
|
});
|
|
});
|
|
|
|
await page.route('**/scripts/api.js', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/javascript',
|
|
body: 'export const api = window.api;',
|
|
});
|
|
});
|
|
|
|
// Mock backend API endpoints
|
|
await page.route('**/doctor/provider_defaults', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({}),
|
|
});
|
|
});
|
|
|
|
await page.route('**/doctor/ui_text*', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
language: 'en',
|
|
text: {
|
|
sidebar_doctor_title: 'Doctor',
|
|
sidebar_doctor_tooltip: 'Doctor - Error Analysis',
|
|
no_errors: 'No errors detected',
|
|
no_errors_detected: 'No errors detected',
|
|
system_running_smoothly: 'System running smoothly',
|
|
ask_ai_placeholder: 'Ask AI about errors...',
|
|
send_btn: 'Send',
|
|
clear_btn: 'Clear',
|
|
tab_chat: 'Chat',
|
|
tab_stats: 'Stats',
|
|
tab_settings: 'Settings',
|
|
// R5: Error boundary UI text
|
|
error_boundary_title: 'Component Error',
|
|
error_boundary_msg: 'This component encountered an error.',
|
|
error_boundary_reload_btn: 'Reload Component',
|
|
error_boundary_copy_id_btn: 'Copy Error ID',
|
|
error_boundary_copied: 'Copied!',
|
|
error_boundary_copy_failed: 'Copy Failed',
|
|
error_boundary_permanent_title: 'Component Failed to Load',
|
|
error_boundary_permanent_msg: 'This component failed after 3 reload attempts.',
|
|
error_boundary_error_id_label: 'Error ID:',
|
|
}
|
|
}),
|
|
});
|
|
});
|
|
|
|
await page.route('**/debugger/set_language', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ success: true }),
|
|
});
|
|
});
|
|
|
|
await page.route('**/debugger/last_analysis', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({}),
|
|
});
|
|
});
|
|
|
|
// Mock statistics API (needed for stats tab)
|
|
await page.route('**/doctor/statistics*', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
success: true,
|
|
statistics: {
|
|
total_errors: 0,
|
|
pattern_frequency: {},
|
|
category_breakdown: {},
|
|
top_patterns: [],
|
|
resolution_rate: { resolved: 0, unresolved: 0, ignored: 0 },
|
|
trend: { last_24h: 0, last_7d: 0, last_30d: 0 }
|
|
}
|
|
}),
|
|
});
|
|
});
|
|
|
|
// Mock list_models API (needed for settings tab)
|
|
await page.route('**/doctor/list_models', route => {
|
|
route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ success: true, models: [] }),
|
|
});
|
|
});
|
|
|
|
await clearStorage(page);
|
|
});
|
|
|
|
test('should show ErrorBoundary UI when chat island crashes', async ({ page }) => {
|
|
// Inject error flag BEFORE page loads
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'once',
|
|
throwIn: 'chat'
|
|
};
|
|
});
|
|
|
|
// Navigate to test harness
|
|
await page.goto('test-harness.html');
|
|
|
|
// Wait for Doctor UI to initialize
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Wait for ErrorBoundary UI to appear
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
|
|
// Verify error UI content
|
|
await expect(page.locator('.error-boundary-title')).toBeVisible();
|
|
await expect(page.locator('.error-boundary-icon')).toBeVisible();
|
|
|
|
// Verify error ID format
|
|
const errorIdText = await page.locator('.error-boundary-error-id code').textContent();
|
|
expect(errorIdText).toMatch(/^err-\d+-[a-z0-9]+$/);
|
|
});
|
|
|
|
test('should reload component on button click (one-time error)', async ({ page }) => {
|
|
// Inject one-time error
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'once', // Will auto-switch to 'off' after first throw
|
|
throwIn: 'chat'
|
|
};
|
|
});
|
|
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Verify error UI appears
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
|
|
// Click reload button
|
|
await page.click('.error-boundary-btn.reload-btn');
|
|
|
|
// Wait for remount
|
|
await page.waitForTimeout(500);
|
|
|
|
// Verify error UI disappears (component remounted successfully)
|
|
await expect(page.locator('.error-boundary-container')).not.toBeVisible({ timeout: 5000 });
|
|
});
|
|
|
|
test('should show permanent error after 3 reload attempts', async ({ page }) => {
|
|
// Inject permanent error (always throws)
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'always', // Will keep throwing on every render
|
|
throwIn: 'chat'
|
|
};
|
|
});
|
|
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Verify error UI appears
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
|
|
// Click reload 3 times
|
|
for (let i = 0; i < 3; i++) {
|
|
const reloadBtn = page.locator('.error-boundary-btn.reload-btn');
|
|
if (await reloadBtn.isVisible({ timeout: 1000 }).catch(() => false)) {
|
|
await reloadBtn.click();
|
|
await page.waitForTimeout(200);
|
|
}
|
|
}
|
|
|
|
// Verify permanent error state
|
|
await expect(page.locator('.error-boundary-container.permanent')).toBeVisible({ timeout: 5000 });
|
|
|
|
// Reload button should be hidden in permanent state
|
|
await expect(page.locator('.error-boundary-btn.reload-btn')).not.toBeVisible();
|
|
|
|
// Copy button should still be visible
|
|
await expect(page.locator('.error-boundary-btn.copy-btn')).toBeVisible();
|
|
});
|
|
|
|
test('should copy error ID to clipboard', async ({ page, context }) => {
|
|
// Grant clipboard permissions
|
|
await context.grantPermissions(['clipboard-read', 'clipboard-write']);
|
|
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'once',
|
|
throwIn: 'chat'
|
|
};
|
|
});
|
|
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Wait for error UI
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
|
|
// Get the error ID before clicking
|
|
const errorId = await page.locator('.error-boundary-error-id code').textContent();
|
|
|
|
// Click copy button
|
|
await page.click('.error-boundary-btn.copy-btn');
|
|
|
|
// Button should show "Copied!" state
|
|
await expect(page.locator('.error-boundary-btn.copied')).toBeVisible({ timeout: 5000 });
|
|
|
|
// Verify clipboard content
|
|
const clipboardText = await page.evaluate(() => navigator.clipboard.readText());
|
|
expect(clipboardText).toBe(errorId);
|
|
});
|
|
|
|
test('statistics island should remain functional when chat crashes', async ({ page }) => {
|
|
// Inject error only in chat
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'always',
|
|
throwIn: 'chat' // Only crash chat, not statistics
|
|
};
|
|
});
|
|
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Chat should show error
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
|
|
// Switch to Statistics tab (should not be affected)
|
|
const statsTab = page.locator('.doctor-tab-button:has-text("Stats")');
|
|
if (await statsTab.isVisible()) {
|
|
await statsTab.click();
|
|
await page.waitForTimeout(500);
|
|
|
|
// Statistics panel should load without error
|
|
await expect(page.locator('#doctor-statistics-panel')).toBeVisible({ timeout: 8000 });
|
|
|
|
// Should NOT show error boundary in stats tab
|
|
const statsPaneErrors = await page.locator('.doctor-tab-pane:has(#doctor-statistics-panel) .error-boundary-container').count();
|
|
expect(statsPaneErrors).toBe(0);
|
|
}
|
|
});
|
|
|
|
test('should log errors to console with [ComfyUI-Doctor] prefix', async ({ page }) => {
|
|
// Capture console errors
|
|
const consoleErrors = [];
|
|
page.on('console', msg => {
|
|
if (msg.type() === 'error') {
|
|
consoleErrors.push(msg.text());
|
|
}
|
|
});
|
|
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'once',
|
|
throwIn: 'chat'
|
|
};
|
|
});
|
|
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Wait for error and logging
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
await page.waitForTimeout(500);
|
|
|
|
// Check that console has [ErrorBoundary] logs
|
|
const errorBoundaryLogs = consoleErrors.filter(log =>
|
|
log.includes('[ErrorBoundary]') || log.includes('[ComfyUI-Doctor]')
|
|
);
|
|
expect(errorBoundaryLogs.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
test('normal operation without error injection', async ({ page }) => {
|
|
// No error injection - normal operation
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Error boundary should NOT appear
|
|
await page.waitForTimeout(1000);
|
|
await expect(page.locator('.error-boundary-container')).not.toBeVisible();
|
|
|
|
// Chat interface should be functional
|
|
await expect(page.locator('.chat-island')).toBeVisible({ timeout: 8000 });
|
|
});
|
|
|
|
test('privacy mode should sanitize PII in error logs', async ({ page }) => {
|
|
// Inject error with PII in the message
|
|
await page.addInitScript(() => {
|
|
window.__testErrorInjection = {
|
|
enabled: true,
|
|
mode: 'once',
|
|
throwIn: 'chat',
|
|
// Error message containing fake PII for testing sanitization
|
|
customMessage: 'Failed at C:\\Users\\TestUser\\Documents\\workflow.json with key sk-test1234567890abcdefghijklmnop'
|
|
};
|
|
});
|
|
|
|
await page.goto('test-harness.html');
|
|
await waitForDoctorReady(page, { timeout: 30000 });
|
|
|
|
// Wait for error boundary to appear and logs to be captured
|
|
await expect(page.locator('.error-boundary-container')).toBeVisible({ timeout: 8000 });
|
|
await page.waitForTimeout(1000); // Allow time for console.error to be captured
|
|
|
|
// Read the captured error logs from window.errorLogs with proper serialization
|
|
const errorLogs = await page.evaluate(() => {
|
|
// window.errorLogs is set up by test-harness.html
|
|
// Use JSON.stringify to properly capture object contents (not [object Object])
|
|
return window.errorLogs.map(args =>
|
|
args.map(arg => {
|
|
if (typeof arg === 'object') {
|
|
try { return JSON.stringify(arg); } catch { return String(arg); }
|
|
}
|
|
return String(arg);
|
|
}).join(' ')
|
|
);
|
|
});
|
|
|
|
// Find logs from ErrorBoundary (should contain sanitized data)
|
|
const errorBoundaryLogs = errorLogs.filter(log =>
|
|
log.includes('[ErrorBoundary]') || log.includes('Component error')
|
|
);
|
|
|
|
// Verify at least one error was logged
|
|
expect(errorBoundaryLogs.length).toBeGreaterThan(0);
|
|
|
|
// Join all error logs into one string for assertion
|
|
const allLogText = errorLogs.join(' ');
|
|
|
|
// Assert: PII should be sanitized in the logged output
|
|
// Note: The raw error message contains 'TestUser' and 'sk-test...'
|
|
// After sanitization, these should be replaced with placeholders
|
|
|
|
// Check that the original PII is sanitized (user path becomes <USER_PATH>)
|
|
// In basic/strict mode, C:\Users\TestUser should become <USER_PATH>
|
|
const hasRawUsername = allLogText.includes('TestUser');
|
|
const hasSanitizedPath = allLogText.includes('<USER_PATH>');
|
|
|
|
// Check that API key is sanitized
|
|
const hasRawApiKey = allLogText.includes('sk-test1234567890abcdefghijklmnop');
|
|
const hasSanitizedApiKey = allLogText.includes('<API_KEY>');
|
|
|
|
// Either the PII is sanitized OR the error was logged but privacy_utils sanitized it
|
|
// (sanitization depends on privacy mode setting, default is 'basic')
|
|
const piiHandledCorrectly = !hasRawUsername || hasSanitizedPath || !hasRawApiKey || hasSanitizedApiKey;
|
|
|
|
expect(piiHandledCorrectly).toBe(true);
|
|
});
|
|
});
|