refactor: centralize source API requests, polling, and path parsing
- Add request_source_response and request_source_json helpers with 429 retry logic - Introduce shared split_path_segments and parse_provider_model_url utilities - Migrate CivitAI, CivArchive, HuggingFace, and ModelList sources to centralized helpers - Add generic pollBackgroundTask utility for frontend progress polling - Add tests for request_source_json, split_path_segments, parse_provider_model_url - Add huggingface-hub to requirements
This commit is contained in:
Dariusz L
@@ -119,3 +119,43 @@ export function sanitizeDescriptionHtml(html) {
|
||||
Array.from(root.childNodes).forEach(child => sanitizeNode(child));
|
||||
return root.innerHTML;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic background task progress poller with token cancellation checks.
|
||||
*/
|
||||
export async function pollBackgroundTask({
|
||||
endpoint,
|
||||
tokenCheck,
|
||||
onProgress,
|
||||
isTerminal,
|
||||
onTerminal,
|
||||
onError,
|
||||
intervalMs = 250,
|
||||
fetchJson,
|
||||
filterIgnoredStatus
|
||||
}) {
|
||||
while (tokenCheck()) {
|
||||
try {
|
||||
const data = await fetchJson(endpoint, { silent: true }, 'Poll background task');
|
||||
|
||||
if (filterIgnoredStatus && filterIgnoredStatus(data)) {
|
||||
await new Promise(resolve => setTimeout(resolve, intervalMs));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tokenCheck()) {
|
||||
onProgress(data);
|
||||
if (isTerminal(data)) {
|
||||
onTerminal(data);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (tokenCheck()) {
|
||||
onError(error);
|
||||
return;
|
||||
}
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, intervalMs));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { app } from "../../../../../scripts/app.js";
|
||||
import { api } from "../../../../../scripts/api.js";
|
||||
import { $el } from "../../../../../scripts/ui.js";
|
||||
import { getSvgIcon } from "../../utils/icon_utils.js";
|
||||
import { escapeHtml, escapeJsString } from "./html_utils.js";
|
||||
import { escapeHtml, escapeJsString, pollBackgroundTask } from "./html_utils.js";
|
||||
|
||||
export const renderFormatMethods = {
|
||||
encodeContextMenuModel(context = null) {
|
||||
@@ -248,69 +248,54 @@ export const renderFormatMethods = {
|
||||
},
|
||||
|
||||
async pollAnalysisProgress(analysisId, token) {
|
||||
while (this._analysisProgressToken === token) {
|
||||
try {
|
||||
const progress = await this.fetchJson(`/model_resolver/analyze-progress/${analysisId}`, {
|
||||
silent: true
|
||||
}, 'Poll analysis progress');
|
||||
if (
|
||||
progress &&
|
||||
this.contentElement &&
|
||||
this._analysisProgressToken === token &&
|
||||
this.activeTab === 'missing'
|
||||
) {
|
||||
await pollBackgroundTask({
|
||||
endpoint: `/model_resolver/analyze-progress/${analysisId}`,
|
||||
tokenCheck: () => this._analysisProgressToken === token,
|
||||
onProgress: (progress) => {
|
||||
if (this.contentElement && this.activeTab === 'missing') {
|
||||
this.contentElement.innerHTML = this.renderAnalysisProgress(progress);
|
||||
if (progress.status === 'completed' || progress.status === 'error') {
|
||||
if (this._analysisProgressToken === token) {
|
||||
this._analysisProgressToken = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
},
|
||||
isTerminal: (progress) => progress.status === 'completed' || progress.status === 'error',
|
||||
onTerminal: () => {
|
||||
if (this._analysisProgressToken === token) {
|
||||
this._analysisProgressToken = null;
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
console.warn('Model Resolver: analysis progress polling failed', error);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
}
|
||||
},
|
||||
intervalMs: 250,
|
||||
fetchJson: this.fetchJson.bind(this)
|
||||
});
|
||||
},
|
||||
|
||||
async pollLoadedModelsProgress(loadedId, token) {
|
||||
while (this._loadedModelsProgressToken === token) {
|
||||
try {
|
||||
const progress = await this.fetchJson(`/model_resolver/loaded-progress/${encodeURIComponent(loadedId)}`, {
|
||||
silent: true
|
||||
}, 'Poll loaded models progress');
|
||||
if (progress?.status === 'unknown') {
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
progress &&
|
||||
this.contentElement &&
|
||||
this._loadedModelsProgressToken === token &&
|
||||
this.activeTab === 'loaded'
|
||||
) {
|
||||
await pollBackgroundTask({
|
||||
endpoint: `/model_resolver/loaded-progress/${encodeURIComponent(loadedId)}`,
|
||||
tokenCheck: () => this._loadedModelsProgressToken === token,
|
||||
filterIgnoredStatus: (progress) => progress?.status === 'unknown',
|
||||
onProgress: (progress) => {
|
||||
if (this.contentElement && this.activeTab === 'loaded') {
|
||||
this.contentElement.innerHTML = this.renderLoadedModelsProgress(progress);
|
||||
if (progress.status === 'completed' || progress.status === 'error') {
|
||||
if (this._loadedModelsProgressToken === token) {
|
||||
this._loadedModelsProgressToken = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
},
|
||||
isTerminal: (progress) => progress.status === 'completed' || progress.status === 'error',
|
||||
onTerminal: () => {
|
||||
if (this._loadedModelsProgressToken === token) {
|
||||
this._loadedModelsProgressToken = null;
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
console.warn('Model Resolver: loaded models progress polling failed', error);
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 250));
|
||||
}
|
||||
},
|
||||
intervalMs: 250,
|
||||
fetchJson: this.fetchJson.bind(this)
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Get display information for the best local match of a missing model
|
||||
* @param {object} missing - The missing model object
|
||||
* @returns {object} Match display metadata
|
||||
*/
|
||||
getLocalMatchDisplayInfo(missing) {
|
||||
const bestMatch = this.getBestLocalMatch?.(missing, 70) || null;
|
||||
|
||||
@@ -4,6 +4,7 @@ import { $el } from "../../../../../scripts/ui.js";
|
||||
import { getSvgIcon } from "../../utils/icon_utils.js";
|
||||
import { LOG_LEVEL as DEFAULT_FRONTEND_LOG_LEVEL } from "../../log_system/config.js";
|
||||
import { logger as frontendLogger } from "../../log_system/logger.js";
|
||||
import { pollBackgroundTask } from "../utils/html_utils.js";
|
||||
const SETTINGS_MAP = [
|
||||
{ serverKey: 'civitai_key', localKey: 'ModelResolver.civitaiApiKey', type: 'string', default: '' },
|
||||
{ serverKey: 'civitai_session_token', localKey: 'ModelResolver.civitaiSessionToken', type: 'string', default: '' },
|
||||
@@ -2067,34 +2068,31 @@ export const optionsMethods = {
|
||||
const pollMetadataBuildProgress = async () => {
|
||||
const progressId = this.metadataBuildProgressId;
|
||||
if (!progressId) return;
|
||||
try {
|
||||
const data = await this.fetchJson(
|
||||
`/model_resolver/metadata-build/progress/${encodeURIComponent(progressId)}`,
|
||||
{ silent: true },
|
||||
'Metadata build progress'
|
||||
);
|
||||
appendMetadataBuildHistory(data);
|
||||
updateMetadataBuildSummary(data);
|
||||
updateMetadataBuildProgressView(data);
|
||||
renderMetadataBuildResults(data);
|
||||
if (isMetadataBuildTerminal(data)) {
|
||||
stopMetadataBuildPolling();
|
||||
stopMetadataBuildPolling();
|
||||
await pollBackgroundTask({
|
||||
endpoint: `/model_resolver/metadata-build/progress/${encodeURIComponent(progressId)}`,
|
||||
tokenCheck: () => this.metadataBuildProgressId === progressId,
|
||||
onProgress: (data) => {
|
||||
appendMetadataBuildHistory(data);
|
||||
updateMetadataBuildSummary(data);
|
||||
updateMetadataBuildProgressView(data);
|
||||
renderMetadataBuildResults(data);
|
||||
},
|
||||
isTerminal: (data) => isMetadataBuildTerminal(data),
|
||||
onTerminal: (data) => {
|
||||
finishMetadataBuild(data);
|
||||
if (this.metadataBuildProgressId === progressId) {
|
||||
this.metadataBuildProgressId = '';
|
||||
}
|
||||
return;
|
||||
}
|
||||
stopMetadataBuildPolling();
|
||||
this.metadataBuildPollTimer = window.setTimeout(() => {
|
||||
pollMetadataBuildProgress();
|
||||
}, 700);
|
||||
} catch (error) {
|
||||
stopMetadataBuildPolling();
|
||||
this.metadataBuildProgressId = '';
|
||||
setMetadataBuildBusy(false);
|
||||
setMetadataBuildStatus(error.message || 'Metadata build progress failed.', 'is-invalid');
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
this.metadataBuildProgressId = '';
|
||||
setMetadataBuildBusy(false);
|
||||
setMetadataBuildStatus(error.message || 'Metadata build progress failed.', 'is-invalid');
|
||||
},
|
||||
intervalMs: 700,
|
||||
fetchJson: this.fetchJson.bind(this)
|
||||
});
|
||||
};
|
||||
|
||||
const runMetadataBuild = async () => {
|
||||
|
||||
Reference in New Issue
Block a user