diff --git a/locales/de.json b/locales/de.json index c80f0366..e2ae305c 100644 --- a/locales/de.json +++ b/locales/de.json @@ -2221,7 +2221,8 @@ "copyReport": "Bericht kopieren", "downloadCsv": "CSV herunterladen", "columnModelName": "Modellname", - "columnError": "Fehler" + "columnError": "Fehler", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "Zusammenfassung des Batch-Downloads", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "Metadaten-Update abgeschlossen", "operationCancelled": "Vorgang vom Benutzer abgebrochen", "operationCancelledPartial": "Vorgang abgebrochen. {success} Elemente verarbeitet.", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "Fehler beim Abrufen der Metadaten: {message}", "bulkMetadataCompleteAll": "Alle {count} {type}s erfolgreich aktualisiert", "bulkMetadataCompletePartial": "{success} von {total} {type}s aktualisiert", diff --git a/locales/en.json b/locales/en.json index 96a8b110..123da099 100644 --- a/locales/en.json +++ b/locales/en.json @@ -2221,7 +2221,8 @@ "copyReport": "Copy Report", "downloadCsv": "Download CSV", "columnModelName": "Model Name", - "columnError": "Error" + "columnError": "Error", + "rateLimitedNotice": "Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "Batch Download Summary", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "Metadata update complete", "operationCancelled": "Operation cancelled by user", "operationCancelledPartial": "Operation cancelled. {success} items processed.", + "metadataFetchRateLimited": "Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "Failed to fetch metadata: {message}", "bulkMetadataCompleteAll": "Successfully refreshed all {count} {type}s", "bulkMetadataCompletePartial": "Refreshed {success} of {total} {type}s", diff --git a/locales/es.json b/locales/es.json index 1da6ae5c..ea67f48d 100644 --- a/locales/es.json +++ b/locales/es.json @@ -2221,7 +2221,8 @@ "copyReport": "Copiar informe", "downloadCsv": "Descargar CSV", "columnModelName": "Nombre del modelo", - "columnError": "Error" + "columnError": "Error", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "Resumen de descarga por lotes", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "Actualización de metadatos completada", "operationCancelled": "Operación cancelada por el usuario", "operationCancelledPartial": "Operación cancelada. {success} elementos procesados.", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "Error al obtener metadatos: {message}", "bulkMetadataCompleteAll": "Actualizados exitosamente todos los {count} {type}s", "bulkMetadataCompletePartial": "Actualizados {success} de {total} {type}s", diff --git a/locales/fr.json b/locales/fr.json index e214afe8..96d242c9 100644 --- a/locales/fr.json +++ b/locales/fr.json @@ -2221,7 +2221,8 @@ "copyReport": "Copier le rapport", "downloadCsv": "Télécharger CSV", "columnModelName": "Nom du modèle", - "columnError": "Erreur" + "columnError": "Erreur", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "Résumé du téléchargement groupé", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "Mise à jour des métadonnées terminée", "operationCancelled": "Opération annulée par l'utilisateur", "operationCancelledPartial": "Opération annulée. {success} éléments traités.", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "Échec de la récupération des métadonnées : {message}", "bulkMetadataCompleteAll": "Actualisation réussie de tous les {count} {type}s", "bulkMetadataCompletePartial": "{success} sur {total} {type}s actualisés", diff --git a/locales/he.json b/locales/he.json index 54660902..4e8f6604 100644 --- a/locales/he.json +++ b/locales/he.json @@ -2221,7 +2221,8 @@ "copyReport": "העתק דוח", "downloadCsv": "הורד CSV", "columnModelName": "שם המודל", - "columnError": "שגיאה" + "columnError": "שגיאה", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "סיכום הורדה בכמות", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "עדכון המטא-נתונים הושלם", "operationCancelled": "הפעולה בוטלה על ידי המשתמש", "operationCancelledPartial": "הפעולה בוטלה. {success} פריטים עובדו.", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "אחזור המטא-נתונים נכשל: {message}", "bulkMetadataCompleteAll": "רועננו בהצלחה כל {count} ה-{type}s", "bulkMetadataCompletePartial": "רועננו {success} מתוך {total} {type}s", diff --git a/locales/ja.json b/locales/ja.json index dff9ad79..4b563987 100644 --- a/locales/ja.json +++ b/locales/ja.json @@ -2221,7 +2221,8 @@ "copyReport": "レポートをコピー", "downloadCsv": "CSVをダウンロード", "columnModelName": "モデル名", - "columnError": "エラー" + "columnError": "エラー", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "バッチダウンロードの概要", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "メタデータ更新完了", "operationCancelled": "ユーザーによって操作がキャンセルされました", "operationCancelledPartial": "操作がキャンセルされました。{success} 個の項目が処理されました。", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "メタデータの取得に失敗しました:{message}", "bulkMetadataCompleteAll": "{count} {type}すべてが正常に更新されました", "bulkMetadataCompletePartial": "{total} {type}のうち {success} が更新されました", diff --git a/locales/ko.json b/locales/ko.json index 7d10c956..abe4edf2 100644 --- a/locales/ko.json +++ b/locales/ko.json @@ -2221,7 +2221,8 @@ "copyReport": "보고서 복사", "downloadCsv": "CSV 다운로드", "columnModelName": "모델 이름", - "columnError": "오류" + "columnError": "오류", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "일괄 다운로드 요약", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "메타데이터 업데이트 완료", "operationCancelled": "사용자에 의해 작업이 취소되었습니다", "operationCancelledPartial": "작업이 취소되었습니다. {success}개 항목이 처리되었습니다.", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "메타데이터 가져오기 실패: {message}", "bulkMetadataCompleteAll": "모든 {count}개 {type}이(가) 성공적으로 새로고침되었습니다", "bulkMetadataCompletePartial": "{total}개 중 {success}개 {type}이(가) 새로고침되었습니다", diff --git a/locales/ru.json b/locales/ru.json index 30e1a504..16d1467e 100644 --- a/locales/ru.json +++ b/locales/ru.json @@ -2221,7 +2221,8 @@ "copyReport": "Копировать отчет", "downloadCsv": "Скачать CSV", "columnModelName": "Имя модели", - "columnError": "Ошибка" + "columnError": "Ошибка", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "Сводка пакетной загрузки", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "Обновление метаданных завершено", "operationCancelled": "Операция отменена пользователем", "operationCancelledPartial": "Операция отменена. Обработано {success} элементов.", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "Не удалось получить метаданные: {message}", "bulkMetadataCompleteAll": "Успешно обновлены все {count} {type}s", "bulkMetadataCompletePartial": "Обновлено {success} из {total} {type}s", diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 02a1fd5c..173961c8 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -2221,7 +2221,8 @@ "copyReport": "复制报告", "downloadCsv": "下载 CSV", "columnModelName": "模型名称", - "columnError": "错误" + "columnError": "错误", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "批量下载摘要", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "元数据更新完成", "operationCancelled": "操作已由用户取消", "operationCancelledPartial": "操作已取消。已处理 {success} 个项目。", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "获取元数据失败:{message}", "bulkMetadataCompleteAll": "全部 {count} 个 {type} 元数据刷新成功", "bulkMetadataCompletePartial": "已刷新 {success}/{total} 个 {type} 元数据", diff --git a/locales/zh-TW.json b/locales/zh-TW.json index c05e1d5a..1e28b38a 100644 --- a/locales/zh-TW.json +++ b/locales/zh-TW.json @@ -2221,7 +2221,8 @@ "copyReport": "複製報告", "downloadCsv": "下載 CSV", "columnModelName": "模型名稱", - "columnError": "錯誤" + "columnError": "錯誤", + "rateLimitedNotice": "[TODO: Translate] Fetch aborted: the metadata provider rate limit was reached. Re-run \"Fetch from Civitai\" later to retry the remaining models." }, "downloadBatchSummary": { "title": "批次下載摘要", @@ -2882,6 +2883,7 @@ "metadataUpdateComplete": "metadata 更新完成", "operationCancelled": "操作已由用戶取消", "operationCancelledPartial": "操作已取消。已處理 {success} 個項目。", + "metadataFetchRateLimited": "[TODO: Translate] Metadata fetch aborted due to rate limiting — updated {success} of {total}, {failed} failed, {skipped} skipped. Re-run Fetch from Civitai later to retry the rest.", "metadataFetchFailed": "取得 metadata 失敗:{message}", "bulkMetadataCompleteAll": "已成功刷新全部 {count} 個 {type}", "bulkMetadataCompletePartial": "已刷新 {success} / {total} 個 {type}", diff --git a/static/css/components/metadata-refresh-result.css b/static/css/components/metadata-refresh-result.css index 40077614..f1241550 100644 --- a/static/css/components/metadata-refresh-result.css +++ b/static/css/components/metadata-refresh-result.css @@ -164,6 +164,25 @@ color: var(--color-success); } +.refresh-rate-limited-notice { + display: flex; + align-items: center; + gap: var(--space-2); + padding: var(--space-3); + margin-bottom: var(--space-3); + background: var(--surface-subtle); + border-left: 4px solid var(--color-warning); + color: var(--lora-text); + border-radius: var(--border-radius-sm); + font-weight: var(--weight-medium); +} + +.refresh-rate-limited-notice i { + font-size: 1.2em; + flex-shrink: 0; + color: var(--color-warning); +} + [data-theme="dark"] .failure-table th { background: var(--lora-surface-solid); } diff --git a/static/js/api/baseModelApi.js b/static/js/api/baseModelApi.js index b593eff3..0566a7ca 100644 --- a/static/js/api/baseModelApi.js +++ b/static/js/api/baseModelApi.js @@ -837,6 +837,22 @@ export class BaseModelApiClient { break; } + case 'rate_limited': { + // Backend aborted the batch after repeated rate limits; + // settle the promise so the loading overlay closes. + let summaryText = `Aborted (rate limited): updated ${data.success} of ${data.processed} ${this.apiConfig.config.displayName}s`; + if (data.failure_count > 0) { + summaryText += ` | ❌ ${data.failure_count} failed`; + } + if (data.skipped_count > 0) { + summaryText += ` | ⏭️ ${data.skipped_count} skipped`; + } + summaryText += ` (⏱ ${data.elapsed_seconds || '?'}s)`; + loading.setStatus(summaryText); + resolve(data); + break; + } + case 'cancelled': loading.setStatus('Operation cancelled by user'); resolve(data); @@ -894,6 +910,17 @@ export class BaseModelApiClient { return; } + if (data.status === 'rate_limited') { + showToast('toast.api.metadataFetchRateLimited', { + success, + total, + failed: data.failure_count || 0, + skipped: data.skipped_count || 0, + }, 'warning'); + this._showFailureDetailsModal(data); + return; + } + this._showFailureDetailsModal(data); } @@ -918,6 +945,11 @@ export class BaseModelApiClient {

${translate('modals.metadataFetchSummary.title', {}, 'Metadata Fetch Summary')}

+ ${data.status === 'rate_limited' ? ` +
+ ${translate('modals.metadataFetchSummary.rateLimitedNotice', {}, 'Fetch aborted: the metadata provider rate limit was reached. Re-run "Fetch from Civitai" later to retry the remaining models.')} +
` : ''} +
diff --git a/tests/frontend/api/baseModelApi.fetchCivitai.test.js b/tests/frontend/api/baseModelApi.fetchCivitai.test.js new file mode 100644 index 00000000..2a8196fd --- /dev/null +++ b/tests/frontend/api/baseModelApi.fetchCivitai.test.js @@ -0,0 +1,233 @@ +import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest'; + +const { + BASE_MODEL_API_MODULE, + STATE_MODULE, + UI_HELPERS_MODULE, + I18N_MODULE, + STORAGE_MODULE, + API_CONFIG_MODULE, + API_FACTORY_MODULE, + SIDEBAR_MANAGER_MODULE, +} = vi.hoisted(() => ({ + BASE_MODEL_API_MODULE: new URL('../../../static/js/api/baseModelApi.js', import.meta.url).pathname, + STATE_MODULE: new URL('../../../static/js/state/index.js', import.meta.url).pathname, + UI_HELPERS_MODULE: new URL('../../../static/js/utils/uiHelpers.js', import.meta.url).pathname, + I18N_MODULE: new URL('../../../static/js/utils/i18nHelpers.js', import.meta.url).pathname, + STORAGE_MODULE: new URL('../../../static/js/utils/storageHelpers.js', import.meta.url).pathname, + API_CONFIG_MODULE: new URL('../../../static/js/api/apiConfig.js', import.meta.url).pathname, + API_FACTORY_MODULE: new URL('../../../static/js/api/modelApiFactory.js', import.meta.url).pathname, + SIDEBAR_MANAGER_MODULE: new URL('../../../static/js/components/SidebarManager.js', import.meta.url).pathname, +})); + +const showToastMock = vi.fn(); +const showCancelButtonMock = vi.fn(); +const setProgressMock = vi.fn(); +const setStatusMock = vi.fn(); +const resetAndReloadMock = vi.fn(); +const showWithProgressMock = vi.fn(async (callback) => { + await callback({ + showCancelButton: showCancelButtonMock, + setProgress: setProgressMock, + setStatus: setStatusMock, + }); +}); + +vi.mock(STATE_MODULE, () => ({ + state: { + loadingManager: { + showWithProgress: showWithProgressMock, + }, + }, + getCurrentPageState: vi.fn(() => ({})), +})); + +vi.mock(UI_HELPERS_MODULE, () => ({ + showToast: showToastMock, +})); + +vi.mock(I18N_MODULE, () => ({ + translate: vi.fn((key, params, fallback) => { + if (fallback) { + return Object.entries(params || {}).reduce( + (text, [name, value]) => text.replaceAll(`{${name}}`, value), + fallback + ); + } + return key; + }), +})); + +vi.mock(STORAGE_MODULE, () => ({ + getStorageItem: vi.fn(), + getSessionItem: vi.fn(), + removeSessionItem: vi.fn(), + saveMapToStorage: vi.fn(), +})); + +vi.mock(API_CONFIG_MODULE, () => ({ + getCompleteApiConfig: vi.fn(() => ({ + endpoints: { fetchAllCivitai: '/api/lm/loras/fetch-all-civitai' }, + config: { displayName: 'LoRA', singularName: 'lora' }, + })), + getCurrentModelType: vi.fn(() => 'loras'), + isValidModelType: vi.fn(() => true), + DOWNLOAD_ENDPOINTS: {}, + HF_ENDPOINTS: {}, + WS_ENDPOINTS: { fetchProgress: '/ws/fetch-progress' }, +})); + +vi.mock(API_FACTORY_MODULE, () => ({ + resetAndReload: resetAndReloadMock, +})); + +vi.mock(SIDEBAR_MANAGER_MODULE, () => ({ + sidebarManager: { refresh: vi.fn() }, +})); + +class FakeWebSocket { + static instances = []; + + constructor(url) { + this.url = url; + this.onopen = null; + this.onerror = null; + this.onmessage = null; + this.close = vi.fn(); + FakeWebSocket.instances.push(this); + queueMicrotask(() => { + this.onopen?.(); + }); + } + + emit(data) { + this.onmessage?.({ data: JSON.stringify(data) }); + } +} + +async function createClient() { + const { BaseModelApiClient } = await import(BASE_MODEL_API_MODULE); + class TestClient extends BaseModelApiClient {} + return new TestClient('loras'); +} + +async function flushMicrotasks() { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe('BaseModelApiClient.fetchCivitaiMetadata rate limiting', () => { + beforeEach(() => { + showToastMock.mockReset(); + showCancelButtonMock.mockReset(); + setProgressMock.mockReset(); + setStatusMock.mockReset(); + resetAndReloadMock.mockReset(); + showWithProgressMock.mockClear(); + FakeWebSocket.instances = []; + vi.stubGlobal('WebSocket', FakeWebSocket); + document.body.innerHTML = ''; + }); + + afterEach(() => { + delete global.fetch; + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + document.body.innerHTML = ''; + }); + + function mockFetchPending() { + let resolveFetch; + global.fetch = vi.fn(() => new Promise((resolve) => { resolveFetch = resolve; })); + return { + resolveOk: (payload = { status: 'success' }) => + resolveFetch({ ok: true, json: async () => payload }), + }; + } + + async function startFetch(client) { + const promise = client.fetchCivitaiMetadata(); + await vi.waitFor(() => { + expect(FakeWebSocket.instances.length).toBe(1); + }); + await flushMicrotasks(); + const socket = FakeWebSocket.instances[0]; + await vi.waitFor(() => { + expect(socket.onmessage).toBeTruthy(); + }); + return { promise, socket }; + } + + it('settles the operation and shows a summary when the backend aborts on rate limiting', async () => { + const fetchControl = mockFetchPending(); + const client = await createClient(); + const { promise, socket } = await startFetch(client); + + expect(socket.url).toBe(`ws://${window.location.host}/ws/fetch-progress`); + + socket.emit({ + status: 'started', total: 10, processed: 0, success: 0, + failure_count: 0, skipped_count: 0, handled: 0, elapsed_seconds: 0, + }); + socket.emit({ + status: 'processing', total: 10, processed: 5, success: 4, + failure_count: 1, skipped_count: 0, handled: 5, + current_name: 'style.safetensors', elapsed_seconds: 5, + }); + socket.emit({ + status: 'rate_limited', total: 10, processed: 7, success: 5, + failure_count: 3, skipped_count: 2, handled: 7, elapsed_seconds: 12, + failures: [ + { name: 'a.safetensors', error: 'Rate limited' }, + { name: 'b.safetensors', error: 'Rate limited' }, + { name: 'c.safetensors', error: 'Rate limited' }, + ], + }); + + fetchControl.resolveOk(); + // Must resolve: before the fix this promise never settled and the + // loading overlay stayed up forever. + await promise; + + const lastStatus = setStatusMock.mock.calls.at(-1)[0]; + expect(lastStatus).toContain('rate limited'); + expect(lastStatus).toContain('5 of 7 LoRAs'); + expect(lastStatus).toContain('3 failed'); + expect(lastStatus).toContain('2 skipped'); + + expect(resetAndReloadMock).toHaveBeenCalledWith(false); + expect(showToastMock).toHaveBeenCalledWith( + 'toast.api.metadataFetchRateLimited', + { success: 5, total: 10, failed: 3, skipped: 2 }, + 'warning' + ); + expect(socket.close).toHaveBeenCalled(); + + const modal = document.getElementById('metadataRefreshResultModal'); + expect(modal).not.toBeNull(); + expect(modal.textContent).toContain('rate limit was reached'); + expect(modal.textContent).toContain('a.safetensors'); + }); + + it('does not show the rate-limited toast for a normal completion', async () => { + const fetchControl = mockFetchPending(); + const client = await createClient(); + const { promise, socket } = await startFetch(client); + + socket.emit({ + status: 'completed', total: 10, processed: 10, success: 10, + failure_count: 0, skipped_count: 0, handled: 10, elapsed_seconds: 20, + }); + + fetchControl.resolveOk(); + await promise; + + expect(showToastMock).not.toHaveBeenCalledWith( + 'toast.api.metadataFetchRateLimited', + expect.anything(), + expect.anything() + ); + const modal = document.getElementById('metadataRefreshResultModal'); + expect(modal).not.toBeNull(); + expect(modal.textContent).toContain('All 10 LoRAs updated successfully!'); + }); +});