diff --git a/core/sources/civarchive.py b/core/sources/civarchive.py index f27ee96..3fc40c2 100644 --- a/core/sources/civarchive.py +++ b/core/sources/civarchive.py @@ -8,6 +8,7 @@ import html import json import os import re +import time from typing import Any, Dict, List, Optional, Tuple from urllib.parse import parse_qs, urljoin, urlparse @@ -122,20 +123,34 @@ def _request_json( timeout: int = 20, ) -> Optional[Dict[str, Any]]: url = f"{CIVARCHIVE_API_URL}{path}" - try: - response = requests.get( - url, - params={k: v for k, v in (params or {}).items() if v is not None}, - headers=REQUEST_HEADERS, - timeout=timeout, - ) - except Exception as e: - log_warn(f"CivArchive API request failed: path={path}, error={e}") - return None + response = None + request_params = {k: v for k, v in (params or {}).items() if v is not None} + for attempt in range(2): + try: + response = requests.get( + url, + params=request_params, + headers=REQUEST_HEADERS, + timeout=timeout, + ) + except Exception as e: + log_warn(f"CivArchive API request failed: path={path}, error={e}") + return None - if response.status_code != 200: + if response.status_code != 429 or attempt == 1: + break + + retry_after = response.headers.get("Retry-After") + try: + delay = float(retry_after) if retry_after else 1.2 + except (TypeError, ValueError): + delay = 1.2 + time.sleep(max(0.5, min(delay, 3.0))) + + if response is None or response.status_code != 200: + status = response.status_code if response is not None else "no-response" log_debug( - f"CivArchive API returned {response.status_code}: path={path}, params={params}" + f"CivArchive API returned {status}: path={path}, params={params}" ) return None @@ -146,6 +161,50 @@ def _request_json( return None +def _extract_model_payload_from_page( + model_id: int, + version_id: Optional[int] = None, + timeout: int = 20, +) -> Optional[Dict[str, Any]]: + params = {"modelVersionId": version_id} if version_id is not None else None + try: + response = requests.get( + f"{CIVARCHIVE_BASE_URL}/models/{model_id}", + params=params, + headers=REQUEST_HEADERS, + timeout=timeout, + ) + except Exception as e: + log_warn(f"CivArchive model page request failed: model_id={model_id}, error={e}") + return None + + if response.status_code != 200: + log_debug( + f"CivArchive model page returned {response.status_code}: model_id={model_id}, version_id={version_id}" + ) + return None + + next_data = _extract_next_data(response.text) + page_props = next_data.get("props", {}).get("pageProps", {}) + if isinstance(page_props.get("data"), dict): + return page_props["data"] + if isinstance(page_props.get("model"), dict): + return page_props["model"] + return page_props if isinstance(page_props, dict) and page_props else None + + +def _request_model_payload( + model_id: int, + version_id: Optional[int] = None, + timeout: int = 20, +) -> Optional[Dict[str, Any]]: + params = {"modelVersionId": version_id} if version_id is not None else None + payload = _request_json(f"/models/{model_id}", params=params, timeout=timeout) + if payload: + return payload + return _extract_model_payload_from_page(model_id, version_id, timeout=timeout) + + def _search_page( query: str, model_type: Optional[str] = None, @@ -290,6 +349,11 @@ def _mirror_from_top_file(file_data: Dict[str, Any]) -> Optional[Dict[str, Any]] "model_version_id": file_data.get("model_version_id") or file_data.get("modelVersionId"), "deletedAt": file_data.get("deletedAt") or file_data.get("deleted_at"), + "is_dead": file_data.get("is_dead") + or file_data.get("isDead") + or file_data.get("likelyDead") + or file_data.get("likely_dead") + or file_data.get("dead"), "is_gated": file_data.get("is_gated"), "is_paid": file_data.get("is_paid"), } @@ -454,6 +518,81 @@ def _collect_download_urls(file_info: Dict[str, Any]) -> List[str]: return non_civitai_urls + civitai_urls +def _normalize_archive_mirrors(file_info: Dict[str, Any]) -> List[Dict[str, Any]]: + mirrors = file_info.get("mirrors") or [] + if not isinstance(mirrors, list): + mirrors = [mirrors] + + normalized: List[Dict[str, Any]] = [] + seen_urls = set() + for mirror in mirrors: + if not isinstance(mirror, dict): + continue + url = _normalize_download_url(mirror.get("url")) + if not url or url in seen_urls: + continue + seen_urls.add(url) + sha256 = mirror.get("sha256") or mirror.get("hash") + deleted_at = mirror.get("deletedAt") or mirror.get("deleted_at") + is_dead = bool( + deleted_at + or mirror.get("is_dead") + or mirror.get("isDead") + or mirror.get("likelyDead") + or mirror.get("likely_dead") + or mirror.get("dead") + or str(mirror.get("status") or "").lower() in {"dead", "deleted", "unavailable", "missing"} + ) + normalized.append( + { + "url": url, + "source": mirror.get("source") or mirror.get("platform"), + "filename": mirror.get("filename") or mirror.get("name") or file_info.get("name"), + "sha256": sha256, + "hash": sha256, + "deleted_at": deleted_at, + "is_dead": is_dead, + "is_gated": mirror.get("is_gated"), + "is_paid": mirror.get("is_paid"), + } + ) + + download_url = _normalize_download_url(file_info.get("downloadUrl")) + if download_url and download_url not in seen_urls: + normalized.append( + { + "url": download_url, + "source": file_info.get("source") or file_info.get("platform"), + "filename": file_info.get("name") or file_info.get("filename"), + "sha256": file_info.get("sha256"), + "hash": file_info.get("sha256"), + "deleted_at": file_info.get("deletedAt") or file_info.get("deleted_at"), + "is_dead": bool( + file_info.get("deletedAt") + or file_info.get("deleted_at") + or file_info.get("is_dead") + or file_info.get("isDead") + or file_info.get("likelyDead") + or file_info.get("likely_dead") + or file_info.get("dead") + or str(file_info.get("status") or "").lower() in {"dead", "deleted", "unavailable", "missing"} + ), + "is_gated": file_info.get("is_gated"), + "is_paid": file_info.get("is_paid"), + } + ) + + non_civitai = [ + mirror for mirror in normalized + if not str(mirror.get("url") or "").startswith(CIVITAI_DOWNLOAD_URL_PREFIXES) + ] + civitai = [ + mirror for mirror in normalized + if str(mirror.get("url") or "").startswith(CIVITAI_DOWNLOAD_URL_PREFIXES) + ] + return non_civitai + civitai + + def _normalize_archive_image(image: Dict[str, Any]) -> Dict[str, Any]: meta = image.get("meta") if isinstance(image.get("meta"), dict) else {} return { @@ -476,6 +615,8 @@ def _normalize_archive_file(file_info: Dict[str, Any], model_id: Optional[int], transformed = _transform_file_entry(file_info) download_urls = _collect_download_urls(transformed) hashes = transformed.get("hashes") if isinstance(transformed.get("hashes"), dict) else {} + metadata = transformed.get("metadata") if isinstance(transformed.get("metadata"), dict) else {} + mirrors = _normalize_archive_mirrors(transformed) return { "id": transformed.get("id"), "name": transformed.get("name"), @@ -486,6 +627,9 @@ def _normalize_archive_file(file_info: Dict[str, Any], model_id: Optional[int], "primary": bool(transformed.get("primary")), "sha256": transformed.get("sha256") or hashes.get("SHA256") or hashes.get("sha256"), "hashes": hashes, + "metadata": metadata, + "mirrors": mirrors, + "mirror_count": len(mirrors), "model_id": model_id or transformed.get("modelId"), "version_id": version_id or transformed.get("modelVersionId"), } @@ -520,7 +664,7 @@ def _normalize_archive_version( return { "id": version_id, "name": version.get("name") or "", - "base_model": version.get("baseModel"), + "base_model": version.get("baseModel") or version.get("base_model") or version.get("baseModelType"), "published_at": version.get("publishedAt") or version.get("createdAt"), "updated_at": version.get("updatedAt"), "description": version.get("description") or "", @@ -540,8 +684,7 @@ def get_civarchive_model_details( if not model_id: return None - params = {"modelVersionId": version_id} if version_id is not None else None - payload = _request_json(f"/models/{model_id}", params=params, timeout=25) + payload = _request_model_payload(model_id, version_id, timeout=25) if not payload and version_id: resolved = resolve_civarchive_model_version(model_id, version_id) if not resolved: @@ -615,40 +758,16 @@ def get_civarchive_model_details( active_version = model_block.get("version") if isinstance(model_block.get("version"), dict) else {} active_version_id = _coerce_int(active_version.get("id")) hydrated_versions: List[Dict[str, Any]] = [] - hydration_cache: Dict[int, Dict[str, Any]] = {} for version_summary in raw_versions: if not isinstance(version_summary, dict): continue current_id = _coerce_int(version_summary.get("id")) - full_version = None if current_id and active_version_id and current_id == active_version_id: - full_version = {**version_summary, **active_version} - elif current_id: - cached = hydration_cache.get(current_id) - if cached is None: - version_payload = _request_json( - f"/models/{model_id}", - params={"modelVersionId": current_id}, - timeout=25, - ) - version_data = _normalize_payload(version_payload or {}) - version_model = ( - version_data.get("model") - if isinstance(version_data.get("model"), dict) - else version_data - ) - cached = ( - version_model.get("version") - if isinstance(version_model.get("version"), dict) - else {} - ) - hydration_cache[current_id] = cached - if cached: - full_version = {**version_summary, **cached} - - hydrated_versions.append(full_version or version_summary) + hydrated_versions.append({**version_summary, **active_version}) + else: + hydrated_versions.append(version_summary) versions = [ _normalize_archive_version(version, context, top_files) @@ -838,7 +957,7 @@ def _build_result_from_payload( "download_url": download_urls[0], "download_urls": download_urls, "size": _size_kb_to_bytes(selected_file.get("sizeKB")), - "base_model": version.get("baseModel"), + "base_model": version.get("baseModel") or version.get("base_model") or version.get("baseModelType"), "tags": tags, "trained_words": trained_words, "images": version.get("images", []), @@ -885,7 +1004,7 @@ def resolve_civarchive_model_version( return None params = {"modelVersionId": version_id} if version_id is not None else None - payload = _request_json(f"/models/{model_id}", params=params) + payload = _request_model_payload(model_id, version_id) if not payload: return None diff --git a/core/sources/civitai.py b/core/sources/civitai.py index 214ba43..a425a08 100644 --- a/core/sources/civitai.py +++ b/core/sources/civitai.py @@ -982,6 +982,7 @@ def _normalize_civitai_file( "primary": bool(file_info.get("primary", False)), "sha256": file_info.get("sha256") or hashes.get("SHA256") or hashes.get("sha256"), "hashes": hashes, + "metadata": file_info.get("metadata") or {}, "model_id": model_id, "version_id": version_id, } diff --git a/web/css/resolver-main.css b/web/css/resolver-main.css index 6ab99be..9c83531 100644 --- a/web/css/resolver-main.css +++ b/web/css/resolver-main.css @@ -3975,6 +3975,9 @@ border-radius: 7px; background: rgba(255,255,255,0.04); } +.mr-model-details-file > .mr-model-details-mirrors { + grid-column: 1 / -1; +} .mr-model-details-file.is-primary { border-color: rgba(114,211,138,0.28); background: rgba(114,211,138,0.08); @@ -3997,12 +4000,150 @@ color: rgba(255,255,255,0.58); font-size: 11px; } +.mr-model-details-file-badges { + display: flex; + flex-wrap: wrap; + gap: 5px; + margin-top: 7px; +} +.mr-model-details-file-badges span { + display: inline-flex; + align-items: center; + min-height: 20px; + max-width: 100%; + padding: 2px 6px; + border-radius: 5px; + background: rgba(255,255,255,0.075); + color: rgba(255,255,255,0.78); + font-size: 10px; + font-weight: 800; + line-height: 1; +} +.mr-model-details-mirrors { + display: grid; + gap: 10px; + margin-top: 2px; + padding-top: 9px; + border-top: 1px solid rgba(255,255,255,0.08); +} +.mr-model-details-mirrors-title { + color: rgba(255,255,255,0.62); + font-size: 10px; + font-weight: 900; + letter-spacing: 0; + text-transform: uppercase; +} +.mr-model-details-mirror-group { + display: grid; + gap: 7px; +} +.mr-model-details-mirror-group-title { + display: inline-flex; + align-items: center; + gap: 7px; + min-width: 0; + color: rgba(255,255,255,0.56); + font-size: 11px; + font-weight: 800; +} +.mr-model-details-mirror-platform-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + flex: 0 0 18px; +} +.mr-model-details-mirror-platform-icon svg { + width: 18px; + height: 18px; +} +.mr-model-details-mirror-list { + display: grid; + gap: 6px; + padding-left: 25px; +} +.mr-model-details-mirror { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 10px; + align-items: center; + min-width: 0; + width: 100%; + min-height: 48px; + padding: 9px 10px; + border: 1px solid rgba(255,255,255,0.07); + border-radius: 6px; + background: rgba(0,0,0,0.15); + color: inherit; + text-align: left; + cursor: pointer; + appearance: none; + -webkit-appearance: none; + transition: background 120ms ease, border-color 120ms ease; +} +.mr-model-details-mirror:hover { + border-color: rgba(255,255,255,0.13); + background: rgba(255,255,255,0.07); +} +.mr-model-details-mirror.is-dead { + border-color: rgba(185,28,28,0.58); + background: rgba(127,29,29,0.36); +} +.mr-model-details-mirror.is-dead:hover { + border-color: rgba(248,113,113,0.74); + background: rgba(153,27,27,0.54); +} +.mr-model-details-mirror-info { + min-width: 0; +} +.mr-model-details-mirror-info strong, +.mr-model-details-mirror-info small { + display: block; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.mr-model-details-mirror-info strong { + color: #f0f0f2; + font-size: 11px; + font-weight: 800; +} +.mr-model-details-mirror-info small { + margin-top: 3px; + color: rgba(255,255,255,0.42); + font-size: 10px; +} +.mr-model-details-mirror.is-dead .mr-model-details-mirror-info small { + color: rgba(254,202,202,0.72); +} +.mr-model-details-mirror-action { + display: inline-flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + color: rgba(255,255,255,0.56); +} +.mr-model-details-mirror-action svg { + width: 17px; + height: 17px; +} +.mr-model-details-mirror.is-dead .mr-model-details-mirror-action { + color: #f87171; +} .mr-model-details-use { min-height: 31px; padding: 0 10px; border-radius: 6px; font-size: 11px; } +.mr-model-details-mirror.mr-model-details-use { + min-height: 48px; + padding: 9px 10px; + font-size: inherit; +} .mr-model-details-loading { display: flex; align-items: center; @@ -4051,6 +4192,12 @@ .mr-model-details-file { grid-template-columns: 1fr; } + .mr-model-details-mirror { + grid-template-columns: 1fr; + } + .mr-model-details-mirror-list { + padding-left: 0; + } } .mr-options-wrap { diff --git a/web/resolver/actions/resolve_download_methods.js b/web/resolver/actions/resolve_download_methods.js index e082431..d061150 100644 --- a/web/resolver/actions/resolve_download_methods.js +++ b/web/resolver/actions/resolve_download_methods.js @@ -1488,13 +1488,18 @@ export const resolveDownloadMethods = { if (civarchiveResult && civarchiveResult.download_url) { const archiveFilename = civarchiveResult.filename || missing.original_path?.split('/').pop()?.split('\\').pop() || ''; const archiveName = civarchiveResult.name || archiveFilename || 'Model'; + const archiveSecondary = [ + archiveName && archiveName !== archiveFilename ? archiveFilename : '', + civarchiveResult.platform || '', + civarchiveResult.base_model || '' + ].filter(Boolean).join(' / '); addRow({ sourceKey: 'civarchive', sourceLabel: 'CivArchive', model: archiveName, version: civarchiveResult.version_name || '', filename: archiveFilename, - secondary: archiveName && archiveName !== archiveFilename ? archiveFilename : (civarchiveResult.platform || civarchiveResult.base_model || ''), + secondary: archiveSecondary, match: this.getSearchResultMatchDisplay(civarchiveResult), size: this.formatSearchResultSize(civarchiveResult), downloadUrl: civarchiveResult.download_url, @@ -1535,13 +1540,17 @@ export const resolveDownloadMethods = { const modelUrl = civitaiResult.url || (civitaiResult.model_id ? `https://civitai.com/models/${civitaiResult.model_id}${civitaiResult.version_id ? `?modelVersionId=${civitaiResult.version_id}` : ''}` : ''); const downloadFilename = missing.civitai_info?.expected_filename || civitaiResult.filename || civitaiResult.name; const modelName = missing.civitai_info?.model_name || civitaiResult.name || downloadFilename || 'Model'; + const civitaiSecondary = [ + civitaiResult.type || '', + civitaiResult.base_model || missing.civitai_info?.base_model || '' + ].filter(Boolean).join(' / '); addRow({ sourceKey: 'civitai', sourceLabel: 'CivitAI', model: modelName, version: missing.civitai_info?.version_name || civitaiResult.version_name || '', filename: downloadFilename, - secondary: civitaiResult.type || civitaiResult.base_model || '', + secondary: civitaiSecondary, match: this.getSearchResultMatchDisplay(civitaiResult), size: this.formatSearchResultSize(civitaiResult), downloadUrl: civitaiResult.download_url, diff --git a/web/resolver/search/search_panel.js b/web/resolver/search/search_panel.js index 0798b01..0d18357 100644 --- a/web/resolver/search/search_panel.js +++ b/web/resolver/search/search_panel.js @@ -1114,6 +1114,12 @@ export const searchPanelMethods = { ); const modelName = modelParts.name || downloadFilename; const fullModelName = this.getVersionedModelName(modelName, modelParts.version); + const baseModel = downloadSource.base_model || missing.civitai_info?.base_model || ''; + const secondaryParts = [ + sourceSecondary, + fullModelName && fullModelName !== downloadFilename ? downloadFilename : '', + baseModel + ].filter(Boolean); return { sourceKey, @@ -1121,7 +1127,7 @@ export const searchPanelMethods = { model: modelName, version: modelParts.version, filename: downloadFilename, - secondary: sourceSecondary || (fullModelName && fullModelName !== downloadFilename ? downloadFilename : ''), + secondary: secondaryParts.join(' / '), match: isFromWorkflow ? { label: 'Provided', className: 'strong' } : this.getSearchResultMatchDisplay(downloadSource, 'Known', 'strong'), diff --git a/web/resolver/views/model_info_methods.js b/web/resolver/views/model_info_methods.js index c535ae4..bd10f17 100644 --- a/web/resolver/views/model_info_methods.js +++ b/web/resolver/views/model_info_methods.js @@ -1186,6 +1186,47 @@ export const modelInfoMethods = { `; }, + async fetchSourceModelDetailsVersion(details, versionId) { + if (!details?._detailsData || !versionId) return; + + try { + const data = details._detailsData; + const tokens = this.getStoredTokens?.() || {}; + const response = await api.fetchApi('/model_resolver/model-details', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + source: data.source || details._sourceModel?.details_source || details._sourceModel?.source, + model_id: data.model_id || details._sourceModel?.model_id || details._sourceModel?.modelId, + version_id: versionId, + civitai_key: tokens.civitai_key || '' + }) + }); + + if (!response.ok) { + throw new Error(`Version details request failed: ${response.status}`); + } + + const fresh = await response.json(); + const freshVersion = fresh.selected_version; + if (!freshVersion?.id) return; + + data.versions = (data.versions || []).map(version => + String(version.id) === String(freshVersion.id) + ? { ...version, ...freshVersion } + : version + ); + data.selected_version = data.versions.find(version => String(version.id) === String(freshVersion.id)) || freshVersion; + if (String(details._selectedVersionId || '') !== String(freshVersion.id)) return; + + details.innerHTML = this.renderSourceModelDetails(data, details._sourceModel || {}); + this.bindTooltips(details); + } catch (error) { + console.error('Model Resolver: failed to fetch source model version details:', error); + this.showNotification?.('Failed to load this model version.', 'error'); + } + }, + renderSourceModelDetails(data = {}, contextModel = {}) { const versions = Array.isArray(data.versions) ? data.versions : []; const selectedVersionId = String(data.selected_version?.id || data.version_id || versions[0]?.id || ''); @@ -1283,6 +1324,8 @@ export const modelInfoMethods = { } return files.map((file, index) => { + const fileMeta = this.getSourceModelFileMeta(file); + const mirrors = this.getSourceModelMirrors(file); const payload = { source: data.source || contextModel.details_source || contextModel.source, model_id: data.model_id || contextModel.model_id, @@ -1299,20 +1342,183 @@ export const modelInfoMethods = { match_type: 'selected', confidence: 100 }; + const mirrorList = mirrors.length > 1 + ? this.renderSourceModelMirrors(mirrors, payload) + : ''; return `