nodes/Azornes-Comfyui-Model-Resolver
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: isolate manual/HuggingFace metadata from stale CivitAI provenance

Browse Source
This commit is contained in:
Dariusz L
2026-07-20 15:34:32 +02:00
parent 16a69f6c47
commit 60ca3a381a
7 changed files with 429 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/downloader.py
+46 -29
View File
@@ -570,26 +570,39 @@ def build_lora_manager_metadata(
source.get("version_description"),
)
civitai_payload = dict(_as_dict(source.get("civitai")))
if model_id and "modelId" not in civitai_payload:
civitai_payload["modelId"] = _coerce_int_or_value(model_id)
if version_id and "id" not in civitai_payload:
civitai_payload["id"] = _coerce_int_or_value(version_id)
if version_name and "name" not in civitai_payload:
civitai_payload["name"] = str(version_name)
if base_model and "baseModel" not in civitai_payload:
civitai_payload["baseModel"] = str(base_model)
if trained_words and "trainedWords" not in civitai_payload:
civitai_payload["trainedWords"] = trained_words
if images and "images" not in civitai_payload:
civitai_payload["images"] = images
if direct_url and "downloadUrl" not in civitai_payload:
civitai_payload["downloadUrl"] = _strip_sensitive_url_params(str(direct_url))
if version_description and "description" not in civitai_payload:
civitai_payload["description"] = str(version_description)
is_civitai_source = source_name in {
"civitai",
"civarchive",
"lora_manager_archive",
} or (
not source_name
and bool(source.get("civitai") or source.get("civitai_details"))
)
civitai_payload = (
dict(_as_dict(source.get("civitai"))) if is_civitai_source else {}
)
if is_civitai_source:
if model_id and "modelId" not in civitai_payload:
civitai_payload["modelId"] = _coerce_int_or_value(model_id)
if version_id and "id" not in civitai_payload:
civitai_payload["id"] = _coerce_int_or_value(version_id)
if version_name and "name" not in civitai_payload:
civitai_payload["name"] = str(version_name)
if base_model and "baseModel" not in civitai_payload:
civitai_payload["baseModel"] = str(base_model)
if trained_words and "trainedWords" not in civitai_payload:
civitai_payload["trainedWords"] = trained_words
if images and "images" not in civitai_payload:
civitai_payload["images"] = images
if direct_url and "downloadUrl" not in civitai_payload:
civitai_payload["downloadUrl"] = _strip_sensitive_url_params(
str(direct_url)
)
if version_description and "description" not in civitai_payload:
civitai_payload["description"] = str(version_description)
files = _as_list(_first_present(selected_version.get("files"), source.get("files")))
if files and "files" not in civitai_payload:
if is_civitai_source and files and "files" not in civitai_payload:
civitai_payload["files"] = files
model_payload = dict(_as_dict(civitai_payload.get("model")))
@@ -602,9 +615,9 @@ def build_lora_manager_metadata(
model_payload["description"] = str(model_description)
if tags and "tags" not in model_payload:
model_payload["tags"] = tags
if model_payload:
if is_civitai_source and model_payload:
civitai_payload["model"] = model_payload
if creator and "creator" not in civitai_payload:
if is_civitai_source and creator and "creator" not in civitai_payload:
civitai_payload["creator"] = creator
metadata_source = _metadata_source_value(source_name, source.get("metadata_source"))
@@ -624,7 +637,10 @@ def build_lora_manager_metadata(
"preview_url": str(preview_url or ""),
"preview_nsfw_level": 0,
"notes": "",
"from_civitai": bool(metadata_source or model_id or version_id),
"from_civitai": bool(
is_civitai_source
and (metadata_source or model_id or version_id or civitai_payload)
),
"civitai": civitai_payload,
"tags": tags,
"modelDescription": str(model_description or ""),
@@ -2298,14 +2314,15 @@ def download_model(
if existing_sha256 == expected_sha256:
message = "This model is already downloaded and matches the source hash."
metadata_path = get_metadata_sidecar_path(dest_path)
if not os.path.exists(metadata_path):
metadata_path = write_lora_manager_metadata(
dest_path,
metadata or {},
category,
url,
) or ""
# Refresh the sidecar even when it already exists. The selected
# source may be more authoritative than metadata left by an
# earlier fuzzy search or manual download attempt.
metadata_path = write_lora_manager_metadata(
dest_path,
metadata or {},
category,
url,
) or ""
size = os.path.getsize(dest_path)
with download_lock:
if download_id in download_progress:
core/sources/huggingface.py
+42 -16
View File
@@ -26,6 +26,7 @@ from ..type_utils import (
extract_file_size,
fetch_remote_file_size_cached,
looks_like_model_file,
normalize_sha256,
prepare_remote_size_probe_url,
)
from .common import build_unified_search_result
@@ -124,8 +125,8 @@ def _author_index_cache_key(author: str, headers: Dict[str, str]) -> str:
return f"{author}::{auth_key}"
def _is_author_index_fresh(index: Optional[Dict[str, Any]]) -> bool:
if not index:
def _is_author_index_fresh(index: Any) -> bool:
if not isinstance(index, dict) or not index:
return False
updated_at = index.get("updated_at")
@@ -142,6 +143,15 @@ def _read_persistent_author_indexes() -> Dict[str, Any]:
return {"version": HF_AUTHOR_INDEX_CACHE_VERSION, "authors": {}}
if not isinstance(data.get("authors"), dict):
data["authors"] = {}
else:
# A short-lived regression stored the raw HuggingFace response list in
# place of the built author index. Ignore malformed entries so they are
# fetched and rebuilt normally.
data["authors"] = {
author: index
for author, index in data["authors"].items()
if isinstance(index, dict)
}
return data
@@ -206,8 +216,6 @@ def _fetch_author_index(author: str, headers: Dict[str, str], limit: int = 200):
if not isinstance(repos, list):
return None
return repos
index = _build_author_index_from_models(author, repos)
log.info(
f"HuggingFace author index refreshed for author={author}: repos={index['repo_count']}, files={index['file_count']}"
@@ -356,13 +364,7 @@ def _build_huggingface_result(
match_type: str,
headers: Optional[Dict[str, str]] = None,
) -> Dict[str, Any]:
lfs_info = file_info.get("lfs") if isinstance(file_info.get("lfs"), dict) else {}
sha256 = (
file_info.get("sha256")
or file_info.get("hash")
or lfs_info.get("sha256")
or lfs_info.get("oid")
)
sha256 = _extract_huggingface_file_sha256(file_info)
download_url = get_huggingface_download_url(repo_id, file_path)
size = extract_file_size(file_info)
if not size:
@@ -382,6 +384,16 @@ def _build_huggingface_result(
)
def _extract_huggingface_file_sha256(file_info: Dict[str, Any]) -> str:
lfs_info = file_info.get("lfs") if isinstance(file_info.get("lfs"), dict) else {}
return normalize_sha256(
file_info.get("sha256")
or file_info.get("hash")
or lfs_info.get("sha256")
or lfs_info.get("oid")
)
def _get_repo_tree(
repo_id: str, headers: Dict[str, str], branch: str = "main"
) -> Optional[List[Dict[str, Any]]]:
@@ -1025,11 +1037,23 @@ def build_huggingface_custom_result(
headers = {}
if token:
headers["Authorization"] = f"Bearer {token}"
size = fetch_remote_file_size_cached(
download_url,
headers=headers,
timeout=10,
)
file_info: Dict[str, Any] = {}
repo_tree = _get_repo_tree(repo_id, headers, branch)
normalized_file_path = file_path.replace("\\", "/").strip("/")
for candidate in repo_tree or []:
candidate_path = str(candidate.get("path") or "").replace("\\", "/").strip("/")
if candidate_path == normalized_file_path:
file_info = candidate
break
size = extract_file_size(file_info)
if not size:
size = fetch_remote_file_size_cached(
download_url,
headers=headers,
timeout=10,
)
sha256 = _extract_huggingface_file_sha256(file_info)
def quote_url_path(val):
return quote(str(val or "").replace("\\", "/"), safe="/")
@@ -1046,6 +1070,8 @@ def build_huggingface_custom_result(
url=download_url,
download_url=download_url,
size=size,
sha256=sha256,
hashes={"SHA256": sha256} if sha256 else {},
match_type="custom_url",
details_source="huggingface",
repo_id=repo_id,
tests/test_downloader_metadata.py
+76
View File
@@ -7,6 +7,7 @@ from unittest.mock import patch
from core import downloader
from core.downloader import (
build_lora_manager_metadata,
download_model,
get_metadata_sidecar_path,
get_progress,
@@ -15,6 +16,34 @@ from core.downloader import (
class DownloaderMetadataSidecarTests(unittest.TestCase):
def test_huggingface_metadata_does_not_create_civitai_identity(self):
with tempfile.TemporaryDirectory() as tmpdir:
model_path = os.path.join(tmpdir, "model.safetensors")
with open(model_path, "wb") as handle:
handle.write(b"abc")
payload = build_lora_manager_metadata(
model_path,
{
"details_source": "huggingface",
"source": "huggingface",
"model_id": "DreamFast/Qwen3-VL-4b-Heretic-ComfyUI",
"filename": "model.safetensors",
"sha256": "a" * 64,
"download_url": (
"https://huggingface.co/DreamFast/repo/resolve/main/"
"model.safetensors"
),
},
category="text_encoders",
)
self.assertEqual("huggingface", payload["source"])
self.assertEqual("a" * 64, payload["sha256"])
self.assertFalse(payload["from_civitai"])
self.assertEqual({}, payload["civitai"])
self.assertIsNone(payload["metadata_source"])
def test_writes_lora_manager_sidecar_and_sanitizes_secrets(self):
with tempfile.TemporaryDirectory() as tmpdir:
model_path = os.path.join(tmpdir, "example.safetensors")
@@ -105,6 +134,53 @@ class DownloaderMetadataSidecarTests(unittest.TestCase):
self.assertTrue(progress["already_exists"])
self.assertIn("already downloaded", progress["message"])
def test_existing_matching_file_replaces_stale_sidecar_provenance(self):
content = b"existing HuggingFace model"
expected_sha256 = hashlib.sha256(content).hexdigest()
with tempfile.TemporaryDirectory() as tmpdir:
model_path = os.path.join(tmpdir, "existing.safetensors")
with open(model_path, "wb") as handle:
handle.write(content)
metadata_path = get_metadata_sidecar_path(model_path)
with open(metadata_path, "w", encoding="utf-8") as handle:
json.dump(
{
"source": "civarchive",
"details_source": "civarchive",
"sha256": "0" * 64,
"hash_status": "completed",
"from_civitai": True,
"civitai": {"modelId": 123},
},
handle,
)
with patch("core.downloader.get_download_directory", return_value=tmpdir):
result = download_model(
"https://huggingface.co/DreamFast/repo/resolve/main/"
"existing.safetensors",
"existing.safetensors",
"text_encoders",
download_id="refreshstalesidecar",
metadata={
"source": "huggingface",
"details_source": "huggingface",
"model_id": "DreamFast/repo",
"filename": "existing.safetensors",
"sha256": expected_sha256,
},
)
with open(metadata_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
self.assertTrue(result["success"])
self.assertTrue(result["already_exists"])
self.assertEqual("huggingface", payload["source"])
self.assertEqual(expected_sha256, payload["sha256"])
self.assertFalse(payload["from_civitai"])
self.assertEqual({}, payload["civitai"])
def test_existing_file_uses_completed_metadata_hash_before_hashing_file(self):
expected_sha256 = "a" * 64
with tempfile.TemporaryDirectory() as tmpdir:
tests/test_downloads_tab_workflow_route.mjs
+122
View File
@@ -567,6 +567,128 @@ test('search suggestion metadata prefers exact matching base model over weaker a
assert.equal(merged.filename, 'snofs_krea_v1.safetensors');
});
test('manual URL download metadata never inherits provider identity or hash from search results', () => {
const getDownloadMetadata = eval(`(${extractMethod(downloadTargetMethodsSource, 'getDownloadMetadata')})`);
const staleHash = 'a'.repeat(64);
const exactHash = 'b'.repeat(64);
const missing = {
original_path: 'QWEN\\qwen3-vl-4b-heretic_int8.safetensors',
category: 'text_encoders',
civitai_info: {
source: 'civarchive',
model_id: 100,
version_id: 200,
sha256: staleHash,
hashes: { SHA256: staleHash },
civitai: { modelId: 100, id: 200 }
},
download_source: {
source: 'civarchive',
model_id: 100,
version_id: 200,
sha256: staleHash
}
};
const source = {
source: 'huggingface',
details_source: 'huggingface',
model_id: 'DreamFast/Qwen3-VL-4b-Heretic-ComfyUI',
filename: 'qwen3-vl-4b-heretic_int8.safetensors',
download_url: 'https://huggingface.co/DreamFast/repo/resolve/main/model.safetensors',
version_url: 'https://huggingface.co/DreamFast/repo/blob/main/model.safetensors',
provided_url: 'https://huggingface.co/DreamFast/repo/blob/main/model.safetensors',
custom_url: true,
url_source: 'custom',
match_type: 'custom_url',
sha256: exactHash,
hashes: { SHA256: exactHash }
};
const dialog = {
getCachedSearchSuggestionData() {
throw new Error('manual URL must not read cached search metadata');
},
getCompatibleCivitaiSearchResult() {
throw new Error('manual URL must not read cached CivitAI metadata');
},
getFilenameFromPath(value = '') {
return String(value).split(/[\\/]/).at(-1) || '';
}
};
const metadata = getDownloadMetadata.call(dialog, missing, source, {
filename: source.filename,
category: 'text_encoders',
url: source.download_url,
openUrl: source.version_url,
pathMetadata: {
filename: source.filename,
category: 'text_encoders'
}
});
assert.equal(metadata.source, 'huggingface');
assert.equal(metadata.model_id, 'DreamFast/Qwen3-VL-4b-Heretic-ComfyUI');
assert.equal(metadata.version_id, '');
assert.equal(metadata.sha256, exactHash);
assert.deepEqual(metadata.hashes, { SHA256: exactHash });
assert.equal(metadata.custom_url, true);
assert.equal(metadata.url_source, 'custom');
assert.equal(metadata.provided_url, source.provided_url);
assert.equal('civitai' in metadata, false);
});
test('manual URL table row preserves the explicit provenance boundary', () => {
const getCustomUrlResultTableRow = eval(`(${extractMethod(searchPanelMethodsSource, 'getCustomUrlResultTableRow')})`);
const result = {
source: 'huggingface',
model_id: 'DreamFast/Qwen3-VL-4b-Heretic-ComfyUI',
filename: 'qwen3-vl-4b-heretic_int8.safetensors',
download_url: 'https://huggingface.co/DreamFast/repo/resolve/main/model.safetensors',
page_url: 'https://huggingface.co/DreamFast/repo/blob/main/model.safetensors',
provided_url: 'https://huggingface.co/DreamFast/repo/blob/main/model.safetensors',
custom_url: true
};
const dialog = {
getFilenameFromPath(value = '') {
return String(value).split(/[\\/]/).at(-1) || '';
},
getMissingDownloadCategory() {
return 'text_encoders';
},
getSourceResultDownloadCategory() {
return 'text_encoders';
},
getMissingModelKey() {
return 'missing-key';
},
getLocalHashMatchIdentitiesForResult() {
return [];
},
getHashMatchLabelForSearchResult() {
return '';
},
getSearchResultMatchDisplay() {
return { label: 'Provided', className: 'strong' };
},
formatSearchResultSize() {
return '';
},
getSearchResultTimestamp() {
return '';
}
};
const row = getCustomUrlResultTableRow.call(dialog, {
original_path: result.filename,
category: 'text_encoders'
}, result);
assert.equal(row.detailsContext.custom_url, true);
assert.equal(row.detailsContext.url_source, 'custom');
assert.equal(row.detailsContext.provided_url, result.provided_url);
assert.equal(row.detailsContext.source, 'huggingface');
});
test('search suggestion does not let a delayed incompatible result replace workflow path context', () => {
const getCachedSearchSuggestionData = eval(`(${extractMethod(downloadTargetMethodsSource, 'getCachedSearchSuggestionData')})`);
const getFirstSearchResult = eval(`(${extractMethod(downloadTargetMethodsSource, 'getFirstSearchResult')})`);
tests/test_huggingface.py
+103
View File
@@ -5,6 +5,10 @@ import tempfile
from pathlib import Path
from unittest.mock import patch
from core.sources.huggingface import (
_fetch_author_index,
_is_author_index_fresh,
_read_persistent_author_indexes,
build_huggingface_custom_result,
parse_huggingface_url,
get_huggingface_download_url,
_normalize_huggingface_size_probe_url,
@@ -15,6 +19,66 @@ from core.type_utils import extract_file_size
class HuggingFaceSourceTests(unittest.TestCase):
def test_fetch_author_index_builds_index_from_provider_repo_list(self):
with patch(
"core.sources.huggingface.execute_provider_json_request",
return_value=[
{
"id": "Comfy-Org/example",
"siblings": [
{
"rfilename": "text_encoders/model.safetensors",
"size": 123,
}
],
}
],
):
index = _fetch_author_index("Comfy-Org", headers={})
self.assertIsInstance(index, dict)
self.assertEqual("Comfy-Org", index["author"])
self.assertEqual(1, index["repo_count"])
self.assertEqual(1, index["file_count"])
self.assertEqual(
"text_encoders/model.safetensors",
index["files"][0]["path"],
)
def test_malformed_list_author_index_is_not_treated_as_fresh(self):
self.assertFalse(_is_author_index_fresh([{"id": "Comfy-Org/example"}]))
def test_persistent_cache_discards_raw_provider_list_entries(self):
with tempfile.TemporaryDirectory() as tmpdir:
index_path = Path(tmpdir) / "huggingface-author-index.json"
index_path.write_text(
json.dumps(
{
"version": 1,
"authors": {
"Comfy-Org": [{"id": "Comfy-Org/example"}],
"Valid": {
"author": "Valid",
"updated_at": 1,
"repo_count": 0,
"file_count": 0,
"repos": [],
"files": [],
},
},
}
),
encoding="utf-8",
)
with patch(
"core.sources.huggingface.HF_AUTHOR_INDEX_CACHE_PATH",
str(index_path),
):
data = _read_persistent_author_indexes()
self.assertNotIn("Comfy-Org", data["authors"])
self.assertIn("Valid", data["authors"])
def test_persistent_author_index_is_pretty_printed(self):
with tempfile.TemporaryDirectory() as tmpdir:
index_path = Path(tmpdir) / "huggingface-author-index.json"
@@ -85,6 +149,45 @@ class HuggingFaceSourceTests(unittest.TestCase):
normalized = _normalize_huggingface_size_probe_url(url)
self.assertEqual(normalized, "https://huggingface.co/user/repo/resolve/main/model.safetensors")
def test_custom_result_uses_sha256_from_exact_huggingface_file(self):
sha256 = "a" * 64
url = (
"https://huggingface.co/DreamFast/Qwen3-VL-4b-Heretic-ComfyUI/"
"blob/main/qwen3-vl-4b-heretic_int8.safetensors"
)
with patch(
"core.sources.huggingface._get_repo_tree",
return_value=[
{
"path": "qwen3-vl-4b-heretic_int8.safetensors",
"size": 123,
"lfs": {"oid": f"sha256:{sha256}", "size": 123},
}
],
):
result = build_huggingface_custom_result(url)
self.assertIsNotNone(result)
self.assertEqual(sha256, result["sha256"])
self.assertEqual({"SHA256": sha256}, result["hashes"])
self.assertEqual(123, result["size"])
self.assertEqual("huggingface", result["source"])
self.assertTrue(result["custom_url"])
def test_custom_result_leaves_hash_empty_when_huggingface_does_not_provide_it(self):
url = "https://huggingface.co/user/repo/blob/main/model.safetensors"
with patch("core.sources.huggingface._get_repo_tree", return_value=None):
with patch(
"core.sources.huggingface.fetch_remote_file_size_cached",
return_value=456,
):
result = build_huggingface_custom_result(url)
self.assertIsNotNone(result)
self.assertEqual("", result["sha256"])
self.assertEqual({}, result["hashes"])
self.assertEqual(456, result["size"])
def test_build_huggingface_search_queries(self):
queries = build_filename_search_queries("some_model_bf16.safetensors")
self.assertIn("some_model", queries)
web/resolver/search/download_target_methods.js
+27 -5
View File
@@ -1341,12 +1341,25 @@ export const downloadTargetMethods = {
getDownloadMetadata(missing = {}, source = {}, options = {}) {
const sourceData = source && typeof source === 'object' ? source : {};
const searchSuggestion = this.getCachedSearchSuggestionData(missing);
const compatibleCivitaiSearch = this.getCompatibleCivitaiSearchResult?.(missing) || {};
const isProvidedUrl = Boolean(
sourceData.custom_url
|| sourceData.url_source === 'custom'
|| sourceData.match_type === 'custom_url'
|| sourceData.provided_url
);
// A URL explicitly selected by the user is an authoritative provenance
// boundary. Never fill its hash, provider IDs, or file details from an
// earlier fuzzy search result belonging to another provider.
const searchSuggestion = isProvidedUrl ? null : this.getCachedSearchSuggestionData(missing);
const compatibleCivitaiSearch = isProvidedUrl
? {}
: (this.getCompatibleCivitaiSearchResult?.(missing) || {});
const inheritedDownloadSource = isProvidedUrl ? {} : (missing?.download_source || {});
const inheritedCivitaiInfo = isProvidedUrl ? {} : (missing?.civitai_info || {});
const merged = {
...(missing?.civitai_info || {}),
...inheritedCivitaiInfo,
...compatibleCivitaiSearch,
...(missing?.download_source || {}),
...inheritedDownloadSource,
...(searchSuggestion || {}),
...sourceData
};
@@ -1359,7 +1372,7 @@ export const downloadTargetMethods = {
|| sourceData.source
|| sourceData.sourceKey
);
const idSource = sourceHasIdentity ? sourceData : (missing?.download_source || {});
const idSource = sourceHasIdentity ? sourceData : inheritedDownloadSource;
const selectedVersion = merged.selected_version || merged.selectedVersion || null;
const pathMetadata = options.pathMetadata || this.getDownloadPathMetadata(missing, sourceData);
const toList = (value) => {
@@ -1437,6 +1450,15 @@ export const downloadTargetMethods = {
if (merged.metadata_source) metadata.metadata_source = merged.metadata_source;
if (merged.is_deleted !== undefined) metadata.is_deleted = Boolean(merged.is_deleted);
if (merged.civitai_deleted !== undefined) metadata.civitai_deleted = Boolean(merged.civitai_deleted);
if (isProvidedUrl) {
metadata.custom_url = true;
metadata.url_source = 'custom';
metadata.provided_url = sourceData.provided_url
|| sourceData.version_url
|| sourceData.page_url
|| sourceData.url
|| '';
}
return metadata;
},
web/resolver/search/search_panel.js
+13 -13
View File
@@ -2086,19 +2086,19 @@ export const searchPanelMethods = {
result.type || '',
result.base_model || ''
].filter(Boolean).join(' / ');
const detailsContext = ['civitai', 'civarchive'].includes(source) && (result.model_id || result.modelId)
? {
...result,
source,
details_source: source,
model_id: result.model_id || result.modelId,
version_id: result.version_id || result.versionId,
name: modelName,
filename,
missing_key: this.getMissingModelKey(missing),
category
}
: null;
const detailsContext = {
...result,
source,
details_source: source,
custom_url: true,
url_source: 'custom',
model_id: result.model_id || result.modelId,
version_id: result.version_id || result.versionId,
name: modelName,
filename,
missing_key: this.getMissingModelKey(missing),
category
};
const localHashMatchIdentities = this.getLocalHashMatchIdentitiesForResult?.(
localHashMatches,
source,