diff --git a/__init__.py b/__init__.py index 663e3c6..597cb4e 100644 --- a/__init__.py +++ b/__init__.py @@ -630,6 +630,60 @@ class ModelResolverExtension: get_local_model_hash_metadata(path, model=model) ) + @routes.post("/model_resolver/local-matches-by-hash") + @json_api_endpoint("local-matches-by-hash") + async def local_matches_by_hash(request): + """Search local model metadata sidecars for a remote SHA256.""" + data = await request.json() + sha256 = normalize_sha256( + data.get("sha256") + or data.get("hash") + or data.get("SHA256") + or "" + ) + if not sha256: + return web.json_response( + {"error": "sha256 is required"}, status=400 + ) + + category = data.get("category", "") + source = str( + data.get("source") + or data.get("hash_lookup_source") + or "download_source" + ).strip().lower().replace("-", "_") + filename = ( + data.get("filename") + or data.get("path") + or data.get("model_name") + or "" + ) + max_matches = to_int(data.get("max_matches"), 20) + force_rescan = to_bool(data.get("force_rescan"), False) + + matches = search_local_matches_by_hash( + sha256, + category=category or None, + max_matches=max_matches, + force_rescan=force_rescan, + ) + enriched_matches = [ + { + **match, + "hash_lookup_source": source or "download_source", + "hash_lookup_filename": filename, + "hash_lookup_sha256": sha256, + } + for match in matches + ] + return web.json_response( + { + "sha256": sha256, + "local_hash_matches": enriched_matches, + "matches": enriched_matches, + } + ) + @routes.post("/model_resolver/open-containing-folder") @json_api_endpoint("open-containing-folder") async def open_containing_folder(request): @@ -2635,37 +2689,27 @@ class ModelResolverExtension: return normalized return "" - def is_hash_lookup_candidate(result): - if not isinstance(result, dict): - return False - try: - confidence = float(result.get("confidence") or 0) - except (TypeError, ValueError): - confidence = 0.0 - if confidence >= 100.0: - return True - match_type = str(result.get("match_type") or "").lower() - if match_type in {"exact", "model_title", "title"}: - return True - return confidence >= 95.0 - def collect_local_hash_matches(payload): matches = [] seen_match_paths = set() - seen_hash_sources = set() - for source_key in ("huggingface", "civitai", "civarchive"): + seen_hashes = set() + for source_key in ( + "huggingface", + "civitai", + "civarchive", + "lora_manager_archive", + "popular", + "model_list", + ): for source_result in iter_result_items(payload.get(source_key)): raise_if_search_cancelled(source_key) - if not is_hash_lookup_candidate(source_result): - continue sha256 = get_result_sha256(source_result) if not sha256: continue - hash_source_key = (source_key, sha256) - if hash_source_key in seen_hash_sources: + if sha256 in seen_hashes: continue - seen_hash_sources.add(hash_source_key) + seen_hashes.add(sha256) self.search_tracker.update( progress_id, @@ -2704,6 +2748,7 @@ class ModelResolverExtension: "hash_lookup_filename": source_result.get("filename") or source_result.get("path") or filename, + "hash_lookup_sha256": sha256, } matches.append(enriched) diff --git a/web/resolver/actions/resolve_download_methods.js b/web/resolver/actions/resolve_download_methods.js index 192285c..7f7c6e7 100644 --- a/web/resolver/actions/resolve_download_methods.js +++ b/web/resolver/actions/resolve_download_methods.js @@ -741,6 +741,88 @@ export const resolveDownloadMethods = { return hashMatches; }, + getExistingLocalHashMatchesForSha(missing = {}, sha256 = '', state = null) { + const hash = this.normalizeSearchResultSha256?.(sha256) + || String(sha256 || '').trim().toLowerCase(); + if (!hash) return []; + + const results = state?.results || {}; + const matches = [ + ...(Array.isArray(results.local_hash_matches) ? results.local_hash_matches : []), + ...(Array.isArray(missing.matches) ? missing.matches : []) + ]; + const seen = new Set(); + const found = []; + matches.forEach(match => { + if (!match || typeof match !== 'object') return; + if (!(match.hash_match || match.match_type === 'hash' || match.hash_lookup_source)) return; + const matchHash = this.normalizeSearchResultSha256?.(this.getLocalMatchHash?.(match) || match.sha256 || match.hash) + || String(this.getLocalMatchHash?.(match) || match.sha256 || match.hash || '').trim().toLowerCase(); + if (matchHash !== hash) return; + const identity = this.getLocalMatchIdentity?.(match) || JSON.stringify(match); + if (seen.has(identity)) return; + seen.add(identity); + found.push(match); + }); + return found; + }, + + async syncRemoteHashMatchesForResult(missing = {}, result = {}, options = {}) { + if (!missing || !result || typeof result !== 'object') return []; + + const sha256 = this.getSearchResultSha256?.(result) || ''; + if (!sha256) return []; + + const workflowKey = options.workflowKey || this.getWorkflowScopedQueueKey?.(); + const state = options.state + || (workflowKey ? this.getSearchStateForWorkflow?.(workflowKey, missing) : null) + || this.searchResultCache?.get?.(this.getMissingSearchKey?.(missing)) + || this.getSearchState?.(missing) + || null; + const existingMatches = this.getExistingLocalHashMatchesForSha(missing, sha256, state); + if (existingMatches.length) { + this.refreshSearchUiForMissing?.(missing, state || null, { workflowKey }); + return existingMatches; + } + + const sourceKey = String( + options.sourceKey + || result.sourceKey + || result.source + || result.details_source + || 'download_source' + ).trim().toLowerCase().replace(/-/g, '_'); + const category = options.category + || result.category + || result.directory + || missing.category + || ''; + const filename = result.filename + || result.downloadFilename + || result.path + || missing.original_path + || ''; + + try { + const data = await this.fetchJson('/model_resolver/local-matches-by-hash', { + method: 'POST', + silent: true, + body: JSON.stringify({ + sha256, + category, + source: sourceKey, + filename, + max_matches: 20 + }) + }, 'Find local models by hash'); + + return this.applyLocalHashMatchesFromSearchResponse?.(missing, data, { workflowKey }) || []; + } catch (error) { + console.warn('Model Resolver: local hash match sync failed:', error); + return []; + } + }, + restoreSearchLocalHashMatchesForMissing(missing) { if (!missing) return missing; @@ -2599,14 +2681,23 @@ export const resolveDownloadMethods = { const civitaiResult = results.civitai ? (Array.isArray(results.civitai) ? results.civitai[0] : results.civitai) : null; const civarchiveResult = results.civarchive ? (Array.isArray(results.civarchive) ? results.civarchive[0] : results.civarchive) : null; const loraManagerArchiveResult = results.lora_manager_archive ? (Array.isArray(results.lora_manager_archive) ? results.lora_manager_archive[0] : results.lora_manager_archive) : null; - const knownDownloadRow = this.getDownloadSourceTableRow(missing, missing.download_source); const localHashMatches = [ ...(Array.isArray(results.local_hash_matches) ? results.local_hash_matches : []), ...(Array.isArray(missing.matches) ? missing.matches.filter(match => match?.hash_lookup_source) : []) ]; + const hashLabelMap = this.getHashMatchLabelMap?.(missing, results) || null; const getHashMatchIdentities = (sourceKey, result) => ( this.getLocalHashMatchIdentitiesForResult?.(localHashMatches, sourceKey, result) || [] ); + const getHashMatchDisplay = (sourceKey, result, fallbackLabel = 'Match', fallbackClass = 'neutral') => { + const identities = getHashMatchIdentities(sourceKey, result); + const hashLabel = this.getHashMatchLabelForSearchResult?.(result, hashLabelMap, identities) || ''; + return { + identities, + match: this.getSearchResultMatchDisplay(result, fallbackLabel, fallbackClass, hashLabel) + }; + }; + const knownDownloadRow = this.getDownloadSourceTableRow(missing, missing.download_source, hashLabelMap); const hasResults = knownDownloadRow || popular || modelListResult || hfResult || civitaiResult || civarchiveResult || loraManagerArchiveResult; const progressHtml = this.renderSearchProgress(state); const hasActiveProgress = this.hasActiveSearchProgress(state); @@ -2662,6 +2753,7 @@ export const resolveDownloadMethods = { addRow(knownDownloadRow); if (popular) { + const popularHash = getHashMatchDisplay('popular', popular, 'Known', 'strong'); const popularFilename = popular.filename || this.getFilenameFromPath(missing.original_path); const missingCategory = this.getMissingDownloadCategory?.(missing, 'checkpoints') || missing.category || 'checkpoints'; const popularSize = popular.size @@ -2678,18 +2770,19 @@ export const resolveDownloadMethods = { model: popular.name || popularFilename, filename: popularFilename, secondary: popular.name && popular.name !== popularFilename ? popularFilename : '', - match: this.getSearchResultMatchDisplay(popular, 'Known', 'strong'), + match: popularHash.match, size: this.formatSearchResultSize({ ...popular, size: popularSize }), downloadUrl: popular.url, downloadFilename: popularFilename, category: popular.directory || missingCategory, openUrl: getModelCardUrl(popular.url), searchedAt: this.getSearchResultTimestamp(popular), - localHashMatchIdentities: getHashMatchIdentities('popular', popular) + localHashMatchIdentities: popularHash.identities }); } if (modelListResult && modelListResult.url) { + const modelListHash = getHashMatchDisplay('model_list', modelListResult); const missingCategory = this.getMissingDownloadCategory?.(missing, 'checkpoints') || missing.category || 'checkpoints'; addRow({ sourceKey: 'model-list', @@ -2697,18 +2790,19 @@ export const resolveDownloadMethods = { model: modelListResult.name || modelListResult.filename, filename: modelListResult.filename, secondary: modelListResult.name && modelListResult.name !== modelListResult.filename ? modelListResult.filename : '', - match: this.getSearchResultMatchDisplay(modelListResult), + match: modelListHash.match, size: this.formatSearchResultSize(modelListResult), downloadUrl: modelListResult.url, downloadFilename: modelListResult.filename, category: modelListResult.directory || missingCategory, openUrl: getModelCardUrl(modelListResult.url), searchedAt: this.getSearchResultTimestamp(modelListResult), - localHashMatchIdentities: getHashMatchIdentities('model_list', modelListResult) + localHashMatchIdentities: modelListHash.identities }); } if (hfResult && hfResult.url) { + const hfHash = getHashMatchDisplay('huggingface', hfResult); const hfRepo = hfResult.repo_id || hfResult.repo || ''; const hfModelUrl = hfRepo ? `https://huggingface.co/${hfRepo}` : getModelCardUrl(hfResult.url); const missingCategory = this.getMissingDownloadCategory?.(missing, 'checkpoints') || missing.category || 'checkpoints'; @@ -2718,18 +2812,19 @@ export const resolveDownloadMethods = { model: hfRepo || hfResult.filename, filename: hfResult.filename, secondary: hfResult.path && hfResult.path !== hfResult.filename ? hfResult.path : '', - match: this.getSearchResultMatchDisplay(hfResult), + match: hfHash.match, size: this.formatSearchResultSize(hfResult), downloadUrl: hfResult.url, downloadFilename: hfResult.filename, category: missingCategory, openUrl: hfModelUrl, searchedAt: this.getSearchResultTimestamp(hfResult), - localHashMatchIdentities: getHashMatchIdentities('huggingface', hfResult) + localHashMatchIdentities: hfHash.identities }); } if (civarchiveResult && civarchiveResult.download_url) { + const civarchiveHash = getHashMatchDisplay('civarchive', civarchiveResult); const archiveFilename = civarchiveResult.filename || this.getFilenameFromPath(missing.original_path); const archiveName = civarchiveResult.name || archiveFilename || 'Model'; const archiveCategory = this.getSourceResultDownloadCategory?.( @@ -2749,14 +2844,14 @@ export const resolveDownloadMethods = { version: civarchiveResult.version_name || '', filename: archiveFilename, secondary: archiveSecondary, - match: this.getSearchResultMatchDisplay(civarchiveResult), + match: civarchiveHash.match, size: this.formatSearchResultSize(civarchiveResult), downloadUrl: civarchiveResult.download_url, downloadFilename: archiveFilename, category: archiveCategory, openUrl: civarchiveResult.url, searchedAt: this.getSearchResultTimestamp(civarchiveResult), - localHashMatchIdentities: getHashMatchIdentities('civarchive', civarchiveResult), + localHashMatchIdentities: civarchiveHash.identities, detailsContext: { ...civarchiveResult, details_source: 'civarchive', @@ -2767,6 +2862,7 @@ export const resolveDownloadMethods = { } if (loraManagerArchiveResult && loraManagerArchiveResult.download_url) { + const loraArchiveHash = getHashMatchDisplay('lora_manager_archive', loraManagerArchiveResult); const archiveFilename = loraManagerArchiveResult.filename || this.getFilenameFromPath(missing.original_path); const archiveName = loraManagerArchiveResult.name || archiveFilename; const archiveCategory = this.getSourceResultDownloadCategory?.( @@ -2781,14 +2877,14 @@ export const resolveDownloadMethods = { version: loraManagerArchiveResult.version_name || '', filename: archiveFilename, secondary: archiveName && archiveName !== archiveFilename ? archiveFilename : '', - match: this.getSearchResultMatchDisplay(loraManagerArchiveResult), + match: loraArchiveHash.match, size: this.formatSearchResultSize(loraManagerArchiveResult), downloadUrl: loraManagerArchiveResult.download_url || '', downloadFilename: archiveFilename, category: archiveCategory, openUrl: loraManagerArchiveResult.url || getModelCardUrl(loraManagerArchiveResult.download_url), searchedAt: this.getSearchResultTimestamp(loraManagerArchiveResult), - localHashMatchIdentities: getHashMatchIdentities('lora_manager_archive', loraManagerArchiveResult), + localHashMatchIdentities: loraArchiveHash.identities, detailsContext: { ...loraManagerArchiveResult, source: 'lora_manager_archive', @@ -2800,6 +2896,7 @@ export const resolveDownloadMethods = { } if (civitaiResult && civitaiResult.download_url) { + const civitaiHash = getHashMatchDisplay('civitai', civitaiResult); const modelUrl = civitaiResult.url || getCivitaiModelUrl(civitaiResult.model_id, civitaiResult.version_id); const downloadFilename = civitaiResult.filename || missing.civitai_info?.expected_filename || civitaiResult.name; const modelName = civitaiResult.name || missing.civitai_info?.model_name || downloadFilename || 'Model'; @@ -2819,14 +2916,14 @@ export const resolveDownloadMethods = { version: civitaiResult.version_name || missing.civitai_info?.version_name || '', filename: downloadFilename, secondary: civitaiSecondary, - match: this.getSearchResultMatchDisplay(civitaiResult), + match: civitaiHash.match, size: this.formatSearchResultSize(civitaiResult), downloadUrl: civitaiResult.download_url, downloadFilename, category: civitaiCategory, openUrl: modelUrl, searchedAt: this.getSearchResultTimestamp(civitaiResult), - localHashMatchIdentities: getHashMatchIdentities('civitai', civitaiResult), + localHashMatchIdentities: civitaiHash.identities, detailsContext: { ...civitaiResult, name: modelName, diff --git a/web/resolver/search/download_target_methods.js b/web/resolver/search/download_target_methods.js index c32630b..e9ecf57 100644 --- a/web/resolver/search/download_target_methods.js +++ b/web/resolver/search/download_target_methods.js @@ -1173,6 +1173,8 @@ export const downloadTargetMethods = { if (merged.hashes && typeof merged.hashes === 'object') metadata.hashes = merged.hashes; if (merged.file_info && typeof merged.file_info === 'object') metadata.file_info = merged.file_info; if (merged.file && typeof merged.file === 'object') metadata.file = merged.file; + if (merged.selected_file && typeof merged.selected_file === 'object') metadata.selected_file = merged.selected_file; + if (merged.selectedFile && typeof merged.selectedFile === 'object') metadata.selected_file = merged.selectedFile; if (Array.isArray(merged.files)) metadata.files = merged.files; if (selectedVersion && typeof selectedVersion === 'object') metadata.selected_version = selectedVersion; if (merged.civitai && typeof merged.civitai === 'object') metadata.civitai = merged.civitai; diff --git a/web/resolver/search/search_panel.js b/web/resolver/search/search_panel.js index 473607a..b33639d 100644 --- a/web/resolver/search/search_panel.js +++ b/web/resolver/search/search_panel.js @@ -1647,10 +1647,11 @@ export const searchPanelMethods = { return typeof result.size === 'number' ? this.formatBytes(result.size) : String(result.size); }, - getSearchResultMatchDisplay(result = {}, fallbackLabel = 'Match', fallbackClass = 'neutral') { + getSearchResultMatchDisplay(result = {}, fallbackLabel = 'Match', fallbackClass = 'neutral', hashLabel = '') { const matchType = String(result.match_type || '').toLowerCase(); - if (result.hash_verified || result.hash_verified_sha256 || matchType === 'hash') { - return { label: 'Hash', className: 'hash' }; + const resolvedHashLabel = String(hashLabel || result.hash_match_label || result.hashLabel || '').trim(); + if (resolvedHashLabel || result.hash_verified || result.hash_verified_sha256 || matchType === 'hash') { + return { label: resolvedHashLabel || 'Hash', className: 'hash' }; } if (matchType === 'exact') { @@ -1707,6 +1708,77 @@ export const searchPanelMethods = { return ''; }, + collectHashLabelMapHashes(missing = {}, results = null) { + const hashes = []; + const seen = new Set(); + const addHash = (value) => { + const hash = this.normalizeSearchResultSha256?.(value) + || String(value || '').trim().toLowerCase(); + if (!hash || !/^[a-f0-9]{64}$/.test(hash) || seen.has(hash)) return; + seen.add(hash); + hashes.push(hash); + }; + const addMatch = (match) => { + if (!match || typeof match !== 'object') return; + if (!(match.hash_match || match.match_type === 'hash' || match.hash_lookup_source)) return; + addHash(this.getLocalMatchHash?.(match) || match.sha256 || match.hash); + }; + const addResult = (result) => { + if (Array.isArray(result)) { + result.forEach(addResult); + return; + } + if (!result || typeof result !== 'object') return; + const matchType = String(result.match_type || '').toLowerCase(); + if (!(result.hash_verified || result.hash_verified_sha256 || matchType === 'hash')) return; + addHash(this.getSearchResultSha256?.(result) || result.sha256 || result.hash); + }; + + (Array.isArray(missing.matches) ? missing.matches : []).forEach(addMatch); + + const stateResults = results + || this.getSearchState?.(missing)?.results + || this.searchResultCache?.get?.(this.getMissingSearchKey?.(missing))?.results + || {}; + (Array.isArray(stateResults.local_hash_matches) ? stateResults.local_hash_matches : []).forEach(addMatch); + + [ + missing.download_source, + stateResults.popular, + stateResults.model_list, + stateResults.huggingface, + stateResults.civitai, + stateResults.civarchive, + stateResults.lora_manager_archive + ].forEach(addResult); + + return hashes; + }, + + getHashMatchLabelMap(missing = {}, results = null) { + const hashes = this.collectHashLabelMapHashes?.(missing, results) || []; + const labelMap = new Map(); + if (!hashes.length) return labelMap; + + const useNumbers = hashes.length > 1; + hashes.forEach((hash, index) => { + labelMap.set(hash, useNumbers ? `Hash ${index + 1}` : 'Hash'); + }); + return labelMap; + }, + + getHashMatchLabelForSearchResult(result = {}, hashLabelMap = null, identities = []) { + const hash = this.getSearchResultSha256?.(result) || ''; + const label = hash && hashLabelMap?.get?.(hash) ? hashLabelMap.get(hash) : ''; + if (!label) return ''; + + const matchType = String(result.match_type || '').toLowerCase(); + const hasLinkedLocalMatch = Array.isArray(identities) && identities.length > 0; + return (hasLinkedLocalMatch || result.hash_verified || result.hash_verified_sha256 || matchType === 'hash') + ? label + : ''; + }, + normalizeHashLookupSourceKey(source = '') { return String(source || '') .trim() @@ -1738,10 +1810,12 @@ export const searchPanelMethods = { if (!match || typeof match !== 'object') return; const matchSource = this.normalizeHashLookupSourceKey(match.hash_lookup_source || ''); - if (!matchSource || matchSource !== normalizedSource) return; - const matchHash = this.normalizeSearchResultSha256(this.getLocalMatchHash(match)); - if (sourceHash && matchHash !== sourceHash) return; + if (sourceHash) { + if (matchHash !== sourceHash) return; + } else if (!matchSource || matchSource !== normalizedSource) { + return; + } addIdentity(this.getLocalMatchIdentity?.(match) || ''); }); @@ -1749,7 +1823,7 @@ export const searchPanelMethods = { return identities; }, - getDownloadSourceTableRow(missing, downloadSource = {}) { + getDownloadSourceTableRow(missing, downloadSource = {}, hashLabelMap = null) { if (!downloadSource?.url) return null; const originalFilename = this.getFilenameFromPath(missing.original_path); @@ -1818,6 +1892,17 @@ export const searchPanelMethods = { const localHashMatches = Array.isArray(missing.matches) ? missing.matches.filter(match => match?.hash_lookup_source) : []; + const localHashMatchIdentities = this.getLocalHashMatchIdentitiesForResult?.( + localHashMatches, + source, + downloadSource + ) || []; + const resolvedHashLabelMap = hashLabelMap || this.getHashMatchLabelMap?.(missing) || null; + const hashMatchLabel = this.getHashMatchLabelForSearchResult?.( + downloadSource, + resolvedHashLabelMap, + localHashMatchIdentities + ) || ''; return { sourceKey, @@ -1826,20 +1911,13 @@ export const searchPanelMethods = { version: modelParts.version, filename: downloadFilename, secondary: secondaryParts.join(' / '), - match: isFromWorkflow - ? { label: 'Provided', className: 'strong' } - : this.getSearchResultMatchDisplay(downloadSource, 'Known', 'strong'), size: this.formatSearchResultSize(downloadSource), downloadUrl: downloadSource.url, downloadFilename, category: rowCategory, openUrl: modelUrl, searchedAt: this.getSearchResultTimestamp(downloadSource), - localHashMatchIdentities: this.getLocalHashMatchIdentitiesForResult?.( - localHashMatches, - source, - downloadSource - ) || [], + localHashMatchIdentities, pathMetadata: rowPathMetadata, downloadMetadata: this.getDownloadMetadata(missing, rowDetailsContext || { ...downloadSource, @@ -1854,7 +1932,10 @@ export const searchPanelMethods = { openUrl: modelUrl, pathMetadata: rowPathMetadata }), - detailsContext: rowDetailsContext + detailsContext: rowDetailsContext, + match: isFromWorkflow + ? { label: 'Provided', className: 'strong' } + : this.getSearchResultMatchDisplay(downloadSource, 'Known', 'strong', hashMatchLabel) }; }, @@ -2137,10 +2218,24 @@ export const searchPanelMethods = { }; }, - renderLocalMatchStatus(match = {}) { + renderLocalMatchStatus(match = {}, hashLabelMap = null) { const confidence = Number(match.confidence || 0); const isHashMatch = Boolean(match.hash_match || match.match_type === 'hash'); - const label = isHashMatch ? 'Hash' : (confidence === 100 ? 'Exact' : 'Partial'); + const model = match.model || {}; + const hashes = model.hashes && typeof model.hashes === 'object' ? model.hashes : {}; + const normalizedHash = String( + match.sha256 + || match.hash + || model.sha256 + || model.hash + || hashes.SHA256 + || hashes.sha256 + || '' + ).trim().toLowerCase(); + const hashLabel = normalizedHash && hashLabelMap?.get?.(normalizedHash) + ? hashLabelMap.get(normalizedHash) + : ''; + const label = isHashMatch ? (hashLabel || 'Hash') : (confidence === 100 ? 'Exact' : 'Partial'); const className = isHashMatch ? 'mr-match-status-hash' : (confidence === 100 ? 'mr-match-status-exact' : 'mr-match-status-partial'); @@ -2178,6 +2273,7 @@ export const searchPanelMethods = { const hasMatches = filteredMatches.length > 0; const perfectMatches = filteredMatches.filter(m => m.confidence === 100); const otherMatches = filteredMatches.filter(m => m.confidence < 100 && m.confidence >= 70); + const hashLabelMap = this.getHashMatchLabelMap?.(missing) || null; let html = ''; @@ -2211,7 +2307,7 @@ export const searchPanelMethods = { html += `