Expand HuggingFace author fallback to multiple known publishers
Support both Comfy-Org and Kijai author indexes as known HuggingFace fallbacks, aggregate their status in the backend, refresh the cached author index, and update the frontend UI and tests.
This commit is contained in:
Dariusz L
+6
-4
@@ -494,11 +494,11 @@ class ModelResolverExtension:
|
||||
build_huggingface_custom_result,
|
||||
check_brave_search_api_key,
|
||||
check_huggingface_token,
|
||||
get_author_fallback_index_status,
|
||||
get_huggingface_download_url,
|
||||
get_huggingface_model_details,
|
||||
get_known_author_fallback_indexes_status,
|
||||
parse_huggingface_url,
|
||||
refresh_author_fallback_index,
|
||||
refresh_known_author_fallback_indexes,
|
||||
search_huggingface_for_file,
|
||||
)
|
||||
from .core.sources.huggingface import (
|
||||
@@ -4535,7 +4535,9 @@ class ModelResolverExtension:
|
||||
@json_api_endpoint("HuggingFace author index status")
|
||||
async def huggingface_author_index_status_route(request):
|
||||
"""Return local HuggingFace author fallback index status."""
|
||||
return web.json_response(get_author_fallback_index_status())
|
||||
return web.json_response(
|
||||
get_known_author_fallback_indexes_status()
|
||||
)
|
||||
|
||||
@routes.post("/model_resolver/huggingface/author-index/refresh")
|
||||
@json_api_endpoint("HuggingFace author index refresh")
|
||||
@@ -4544,7 +4546,7 @@ class ModelResolverExtension:
|
||||
data = await request.json()
|
||||
hf_token = data.get("hf_token", "")
|
||||
result = await asyncio.to_thread(
|
||||
refresh_author_fallback_index, hf_token or None
|
||||
refresh_known_author_fallback_indexes, hf_token or None
|
||||
)
|
||||
clear_huggingface_search_cache()
|
||||
return web.json_response(result)
|
||||
|
||||
@@ -33,7 +33,7 @@ from ..type_utils import (
|
||||
from .common import build_unified_search_result
|
||||
|
||||
HF_API_URL = "https://huggingface.co/api"
|
||||
HF_AUTHOR_FALLBACKS = ["Comfy-Org"]
|
||||
HF_AUTHOR_FALLBACKS = ["Comfy-Org", "Kijai"]
|
||||
BRAVE_SEARCH_API_URL = "https://api.search.brave.com/res/v1/web/search"
|
||||
HF_AUTHOR_INDEX_CACHE_TTL_SECONDS = 24 * 60 * 60
|
||||
HF_AUTHOR_INDEX_CACHE_VERSION = 1
|
||||
@@ -284,6 +284,33 @@ def get_author_fallback_index_status(author: str = "Comfy-Org") -> Dict[str, Any
|
||||
}
|
||||
|
||||
|
||||
def get_known_author_fallback_indexes_status() -> Dict[str, Any]:
|
||||
"""Return aggregate status for all known HuggingFace author indexes."""
|
||||
author_statuses = [
|
||||
get_author_fallback_index_status(author) for author in HF_AUTHOR_FALLBACKS
|
||||
]
|
||||
updated_times = [
|
||||
status["updated_at"]
|
||||
for status in author_statuses
|
||||
if isinstance(status.get("updated_at"), (int, float))
|
||||
]
|
||||
cached_author_count = sum(bool(status["exists"]) for status in author_statuses)
|
||||
fully_cached = cached_author_count == len(author_statuses)
|
||||
|
||||
return {
|
||||
"authors": author_statuses,
|
||||
"author_count": len(author_statuses),
|
||||
"cached_author_count": cached_author_count,
|
||||
"exists": cached_author_count > 0,
|
||||
"fully_cached": fully_cached,
|
||||
"updated_at": max(updated_times) if updated_times else None,
|
||||
"stale": not fully_cached
|
||||
or any(bool(status["stale"]) for status in author_statuses),
|
||||
"repo_count": sum(int(status["repo_count"]) for status in author_statuses),
|
||||
"file_count": sum(int(status["file_count"]) for status in author_statuses),
|
||||
}
|
||||
|
||||
|
||||
def refresh_author_fallback_index(
|
||||
token: Optional[str] = None, author: str = "Comfy-Org"
|
||||
) -> Dict[str, Any]:
|
||||
@@ -297,6 +324,25 @@ def refresh_author_fallback_index(
|
||||
return status
|
||||
|
||||
|
||||
def refresh_known_author_fallback_indexes(
|
||||
token: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Force refresh all known public HuggingFace author indexes."""
|
||||
refresh_results = [
|
||||
refresh_author_fallback_index(token, author) for author in HF_AUTHOR_FALLBACKS
|
||||
]
|
||||
status = get_known_author_fallback_indexes_status()
|
||||
failed_authors = [
|
||||
result["author"] for result in refresh_results if not result.get("success")
|
||||
]
|
||||
status["success"] = not failed_authors
|
||||
status["persisted"] = all(
|
||||
bool(result.get("persisted")) for result in refresh_results
|
||||
)
|
||||
status["failed_authors"] = failed_authors
|
||||
return status
|
||||
|
||||
|
||||
def parse_huggingface_url(url: str) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
Parse a HuggingFace URL to extract repo and filename.
|
||||
|
||||
+6615
-947
File diff suppressed because it is too large
Load Diff
@@ -5,10 +5,12 @@ import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from core.sources.huggingface import (
|
||||
HF_AUTHOR_FALLBACKS,
|
||||
_fetch_author_index,
|
||||
_is_author_index_fresh,
|
||||
_read_persistent_author_indexes,
|
||||
build_huggingface_custom_result,
|
||||
get_known_author_fallback_indexes_status,
|
||||
get_huggingface_model_details,
|
||||
parse_huggingface_url,
|
||||
get_huggingface_download_url,
|
||||
@@ -20,6 +22,51 @@ from core.type_utils import extract_file_size
|
||||
|
||||
class HuggingFaceSourceTests(unittest.TestCase):
|
||||
|
||||
def test_known_author_fallbacks_include_comfy_org_and_kijai(self):
|
||||
self.assertEqual(["Comfy-Org", "Kijai"], HF_AUTHOR_FALLBACKS)
|
||||
|
||||
def test_known_author_status_reports_each_author_file_count(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
index_path = Path(tmpdir) / "huggingface-author-index.json"
|
||||
index_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"version": 1,
|
||||
"authors": {
|
||||
"Comfy-Org": {
|
||||
"author": "Comfy-Org",
|
||||
"updated_at": 1,
|
||||
"repo_count": 2,
|
||||
"file_count": 3,
|
||||
"repos": [],
|
||||
"files": [],
|
||||
},
|
||||
"Kijai": {
|
||||
"author": "Kijai",
|
||||
"updated_at": 2,
|
||||
"repo_count": 4,
|
||||
"file_count": 5,
|
||||
"repos": [],
|
||||
"files": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with patch(
|
||||
"core.sources.huggingface.HF_AUTHOR_INDEX_CACHE_PATH",
|
||||
str(index_path),
|
||||
):
|
||||
status = get_known_author_fallback_indexes_status()
|
||||
|
||||
self.assertEqual(
|
||||
{"Comfy-Org": 3, "Kijai": 5},
|
||||
{item["author"]: item["file_count"] for item in status["authors"]},
|
||||
)
|
||||
self.assertEqual(8, status["file_count"])
|
||||
self.assertTrue(status["fully_cached"])
|
||||
|
||||
def test_fetch_author_index_builds_index_from_provider_repo_list(self):
|
||||
with patch(
|
||||
"core.sources.huggingface.execute_provider_json_request",
|
||||
|
||||
@@ -14,6 +14,10 @@ const modelInfoSource = await readFile(
|
||||
new URL('../web/resolver/views/model_info_methods.js', import.meta.url),
|
||||
'utf8'
|
||||
);
|
||||
const optionsSource = await readFile(
|
||||
new URL('../web/resolver/views/options_methods.js', import.meta.url),
|
||||
'utf8'
|
||||
);
|
||||
const resolveDownloadSource = await readFile(
|
||||
new URL('../web/resolver/actions/resolve_download_methods.js', import.meta.url),
|
||||
'utf8'
|
||||
@@ -23,6 +27,13 @@ const resolverCssSource = await readFile(
|
||||
'utf8'
|
||||
);
|
||||
|
||||
test('known HuggingFace fallback shows Comfy-Org and Kijai index counts', () => {
|
||||
assert.match(optionsSource, /Use known HuggingFace fallback/);
|
||||
assert.match(optionsSource, /<span>Comfy-Org files<\/span>/);
|
||||
assert.match(optionsSource, /<span>Kijai files<\/span>/);
|
||||
assert.match(optionsSource, /Array\.isArray\(data\.authors\)/);
|
||||
});
|
||||
|
||||
test('HuggingFace download rows expose the Show More action', () => {
|
||||
assert.match(
|
||||
searchPanelSource,
|
||||
|
||||
@@ -6687,6 +6687,10 @@ a.mr-image-preview-resource-name:hover {
|
||||
background: rgba(255, 255, 255, 0.03);
|
||||
}
|
||||
|
||||
#mr-options-hf-index-authors {
|
||||
display: contents;
|
||||
}
|
||||
|
||||
.mr-options-db-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
@@ -628,7 +628,7 @@ export const optionsMethods = {
|
||||
</div>
|
||||
<label class="mr-options-toggle-row">
|
||||
<div class="mr-options-toggle-copy">
|
||||
<span class="mr-options-toggle-title">Use Comfy-Org fallback <span class="mr-tooltip-badge" data-tooltip="Checks Comfy-Org repositories directly. Useful for ComfyUI model packs that normal Hugging Face search may miss.">?</span></span>
|
||||
<span class="mr-options-toggle-title">Use known HuggingFace fallback <span class="mr-tooltip-badge" data-tooltip="Checks cached indexes from known Hugging Face publishers such as Comfy-Org and Kijai. Useful for model files that normal Hugging Face search may miss.">?</span></span>
|
||||
</div>
|
||||
<span class="mr-options-toggle-control">
|
||||
<input id="mr-options-hf-use-comfy-org-fallback" class="mr-options-switch-input" type="checkbox" ${tokens.hf_use_comfy_org_fallback ? 'checked' : ''}>
|
||||
@@ -637,9 +637,15 @@ export const optionsMethods = {
|
||||
</label>
|
||||
<div class="mr-options-dependent-block">
|
||||
<div class="mr-options-db-summary">
|
||||
<div class="mr-options-db-row">
|
||||
<span>Comfy-Org files</span>
|
||||
<strong id="mr-options-hf-index-count">Loading...</strong>
|
||||
<div id="mr-options-hf-index-authors">
|
||||
<div class="mr-options-db-row">
|
||||
<span>Comfy-Org files</span>
|
||||
<strong>Loading...</strong>
|
||||
</div>
|
||||
<div class="mr-options-db-row">
|
||||
<span>Kijai files</span>
|
||||
<strong>Loading...</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mr-options-db-row">
|
||||
<span>Last refresh</span>
|
||||
@@ -650,9 +656,9 @@ export const optionsMethods = {
|
||||
<strong id="mr-options-hf-index-state">Checking local index...</strong>
|
||||
</div>
|
||||
</div>
|
||||
<div id="mr-options-hf-index-message" class="mr-options-db-message">Comfy-Org fallback uses a cached HuggingFace file index.</div>
|
||||
<div id="mr-options-hf-index-message" class="mr-options-db-message">Known HuggingFace fallback uses cached file indexes.</div>
|
||||
<div class="mr-options-db-actions">
|
||||
<button id="mr-options-hf-index-refresh" type="button" class="mr-btn mr-btn-secondary">${getSvgIcon('refreshCw')} Refresh Comfy-Org Index</button>
|
||||
<button id="mr-options-hf-index-refresh" type="button" class="mr-btn mr-btn-secondary">${getSvgIcon('refreshCw')} Refresh Known Indexes</button>
|
||||
</div>
|
||||
</div>
|
||||
<label class="mr-options-toggle-row">
|
||||
@@ -972,7 +978,7 @@ export const optionsMethods = {
|
||||
const baseModelsMessageEl = this.contentElement.querySelector('#mr-options-base-models-message');
|
||||
const baseModelsCheckBtn = this.contentElement.querySelector('#mr-options-base-models-check');
|
||||
const baseModelsUpdateBtn = this.contentElement.querySelector('#mr-options-base-models-update');
|
||||
const hfIndexCountEl = this.contentElement.querySelector('#mr-options-hf-index-count');
|
||||
const hfIndexAuthorsEl = this.contentElement.querySelector('#mr-options-hf-index-authors');
|
||||
const hfIndexUpdatedEl = this.contentElement.querySelector('#mr-options-hf-index-updated');
|
||||
const hfIndexStateEl = this.contentElement.querySelector('#mr-options-hf-index-state');
|
||||
const hfIndexMessageEl = this.contentElement.querySelector('#mr-options-hf-index-message');
|
||||
@@ -2641,10 +2647,22 @@ export const optionsMethods = {
|
||||
});
|
||||
|
||||
const renderHfIndexStatus = (data = {}) => {
|
||||
if (hfIndexCountEl) {
|
||||
hfIndexCountEl.textContent = Number.isFinite(Number(data.file_count))
|
||||
? Number(data.file_count).toLocaleString()
|
||||
: '-';
|
||||
const authorStatuses = Array.isArray(data.authors) && data.authors.length
|
||||
? data.authors
|
||||
: [data];
|
||||
if (hfIndexAuthorsEl) {
|
||||
hfIndexAuthorsEl.innerHTML = authorStatuses.map((status) => {
|
||||
const author = escapeHtml(String(status.author || 'Unknown'));
|
||||
const fileCount = Number.isFinite(Number(status.file_count))
|
||||
? Number(status.file_count).toLocaleString()
|
||||
: '-';
|
||||
return `
|
||||
<div class="mr-options-db-row">
|
||||
<span>${author} files</span>
|
||||
<strong>${fileCount}</strong>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
if (hfIndexUpdatedEl) {
|
||||
hfIndexUpdatedEl.textContent = data.updated_at
|
||||
@@ -2654,6 +2672,8 @@ export const optionsMethods = {
|
||||
if (hfIndexStateEl) {
|
||||
if (!data.exists) {
|
||||
hfIndexStateEl.textContent = 'Not cached';
|
||||
} else if (data.fully_cached === false) {
|
||||
hfIndexStateEl.textContent = 'Partially cached';
|
||||
} else if (data.stale) {
|
||||
hfIndexStateEl.textContent = 'Refresh due';
|
||||
} else {
|
||||
@@ -2662,11 +2682,13 @@ export const optionsMethods = {
|
||||
}
|
||||
if (hfIndexMessageEl) {
|
||||
if (!data.exists) {
|
||||
hfIndexMessageEl.textContent = 'The first Comfy-Org fallback search will build the local HuggingFace file index.';
|
||||
hfIndexMessageEl.textContent = 'The first known-source fallback search will build the local HuggingFace file indexes.';
|
||||
} else if (data.fully_cached === false) {
|
||||
hfIndexMessageEl.textContent = `${Number(data.cached_author_count || 0).toLocaleString()} of ${Number(data.author_count || authorStatuses.length).toLocaleString()} known-source indexes are cached.`;
|
||||
} else if (data.stale) {
|
||||
hfIndexMessageEl.textContent = `Index has ${Number(data.repo_count || 0).toLocaleString()} repos and should be refreshed.`;
|
||||
hfIndexMessageEl.textContent = `Indexes have ${Number(data.repo_count || 0).toLocaleString()} repos and should be refreshed.`;
|
||||
} else {
|
||||
hfIndexMessageEl.textContent = `Index has ${Number(data.repo_count || 0).toLocaleString()} repos and is used before per-repo fallback checks.`;
|
||||
hfIndexMessageEl.textContent = `Indexes have ${Number(data.repo_count || 0).toLocaleString()} repos and are used before per-repo fallback checks.`;
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -2686,7 +2708,7 @@ export const optionsMethods = {
|
||||
stateEl: hfIndexStateEl,
|
||||
messageEl: hfIndexMessageEl,
|
||||
loadingText: 'Refreshing...',
|
||||
startingMessageText: 'Downloading Comfy-Org file index from HuggingFace...',
|
||||
startingMessageText: 'Downloading known-source file indexes from HuggingFace...',
|
||||
endpoint: '/model_resolver/huggingface/author-index/refresh',
|
||||
fetchOptions: {
|
||||
method: 'POST',
|
||||
@@ -2702,7 +2724,7 @@ export const optionsMethods = {
|
||||
successCallback: (data) => {
|
||||
renderHfIndexStatus(data);
|
||||
if (hfIndexStateEl) hfIndexStateEl.textContent = 'Refreshed';
|
||||
this.showNotification('HuggingFace Comfy-Org index refreshed', 'success');
|
||||
this.showNotification('Known HuggingFace indexes refreshed', 'success');
|
||||
},
|
||||
errorFallbackText: 'Failed to refresh HuggingFace index.',
|
||||
notificationErrorText: 'HuggingFace index refresh failed'
|
||||
|
||||
Reference in New Issue
Block a user