nodes/willmiao-ComfyUI-Lora-Manager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

merge: fix empty-file SHA256 handling (#1162)

Browse Source 
Stop registering zero-byte models, repair the empty-string placeholder on scan
and fetch, refuse it in CivArchive/the archive DB/ModelScope linking, and
rebuild the required local fields when relinking without a sidecar.
This commit is contained in:
Will Miao
2026-10-25 19:49:21 +08:00
parent a7d2a921a1 bf63e5d62d
commit 6dc4956ec7
17 changed files with 691 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/services/checkpoint_scanner.py
+12
View File
@@ -223,6 +223,18 @@ class CheckpointScanner(ModelScanner):
# Calculate hash
logger.info(f"Calculating hash for checkpoint: {file_path}")
sha256 = await calculate_sha256(real_path)
if not sha256:
# Nothing was read (the file is empty or was truncated while
# open): there is no identity to store. Keep the lazy-hash
# state so a later pass retries once the file has content, and
# never index the empty-string placeholder (issue #1162).
logger.warning(
"No bytes read from checkpoint %s; keeping hash pending",
file_path,
)
metadata.hash_status = "pending"
await MetadataManager.save_metadata(file_path, metadata)
return None
# Update metadata with hash
metadata.sha256 = sha256
py/services/civarchive_client.py
+17 -1
View File
@@ -7,6 +7,7 @@ import logging
import asyncio
from copy import deepcopy
from typing import Any, Optional, Dict, Tuple, List, cast
from ..utils.constants import is_empty_placeholder_hash
from .connectivity_guard import is_expected_offline_error
from .model_metadata_provider import CivArchiveModelMetadataProvider, ModelMetadataProviderManager
from .downloader import get_downloader
@@ -59,7 +60,10 @@ class CivArchiveClient:
file_data: Dict[str, Any],
) -> None:
"""Record a resolvable source URL from a foreign-ecosystem file entry"""
if not expected_hash:
if not expected_hash or is_empty_placeholder_hash(expected_hash):
# The placeholder identifies no file, so any repo "sharing" it
# (e.g. a repository that publishes a zero-byte upload) must not
# become the link target for a local model (issue #1162).
return
url = file_data.get("url")
if not isinstance(url, str) or not url.strip():
@@ -373,6 +377,11 @@ class CivArchiveClient:
"""Fallback to fetch version data when only file metadata is available"""
if not isinstance(payload, dict):
return None
if expected_hash and is_empty_placeholder_hash(expected_hash):
# A file-entry list returned for the zero-byte digest describes no
# real file; resolving (or remembering) any of its entries would
# attribute an unrelated model to the caller (issue #1162).
return None
data = self._normalize_payload(payload)
files = data.get("files") or payload.get("files") or []
if not isinstance(files, list):
@@ -425,6 +434,13 @@ class CivArchiveClient:
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
"""Find model by SHA256 hash value using CivArchive API"""
if is_empty_placeholder_hash(model_hash):
# CivArchive's by-hash index carries a large pile of unrelated
# foreign files under the zero-byte digest (and records one of them
# as a foreign source ref), so the placeholder must never be
# resolved — it once linked seven unrelated LoRAs to one ModelScope
# repository (issue #1162).
return None, "Model not found"
try:
payload, error = await self._request_json(f"/sha256/{model_hash.lower()}")
# Treat a missing payload as an error even when the error string is
py/services/metadata_sync_service.py
+132 -1
View File
@@ -10,9 +10,12 @@ from typing import Any, Awaitable, Callable, Dict, Iterable, Optional, Protocol
from ..services.settings_manager import SettingsManager
from ..utils.civitai_utils import resolve_license_payload
from ..utils.constants import is_empty_placeholder_hash
from ..utils.file_utils import calculate_sha256, find_preview_file, normalize_path
from ..utils.metadata_manager import MetadataManager
from ..utils.model_utils import determine_base_model
from ..utils.models import autov3_from_civitai_files
from ..utils.sidecar_paths import get_metadata_path
from ..utils.sidecar_paths import get_metadata_path, get_preview_dir
from .connectivity_guard import OFFLINE_FRIENDLY_MESSAGE, is_expected_offline_error
from .errors import MetadataPersistError, RateLimitError
from .model_metadata_provider import _LOCAL_PROVIDER_LABELS
@@ -277,6 +280,66 @@ class MetadataSyncService:
"Failed to unblock example image failures for %s: %s", sha256, exc
)
async def _repair_placeholder_hash(
self,
*,
file_path: str,
model_data: Dict[str, Any],
update_cache_func: Callable[[str, str, Dict[str, Any]], Awaitable[bool]],
) -> tuple[Optional[str], Optional[str]]:
"""Re-hash a model whose stored SHA256 is the empty-file placeholder.
The digest of zero bytes identifies no file, and both CivitAI and
CivArchive indexes carry polluted entries for it (issue #1162). Only a
real hash of the model's current content may reach the provider chain,
so the file is read once here and the repaired value is written to the
sidecar and the cache before the lookup.
Returns ``(new_hash, None)`` when the hash was repaired, or
``(None, error)`` when there is nothing to hash yet. The caller must
not run a lookup in the latter case.
"""
if not file_path:
return None, "Model file path is missing; metadata fetch skipped"
try:
if os.path.getsize(file_path) == 0:
return None, "Model file is empty (0 bytes); metadata fetch skipped"
except OSError:
return None, "Model file not found; metadata fetch skipped"
try:
new_hash = await calculate_sha256(file_path)
except Exception as exc:
logger.error("Failed to recompute SHA256 for %s: %s", file_path, exc)
return None, "Failed to compute SHA256 hash for model"
if not new_hash or is_empty_placeholder_hash(new_hash):
return None, "Failed to compute SHA256 hash for model"
new_hash = new_hash.lower()
logger.info(
"Recomputed SHA256 for %s: the stored hash was the empty-file placeholder",
file_path,
)
model_data["sha256"] = new_hash
model_data["hash_status"] = "completed"
data_to_save = model_data.copy()
data_to_save.pop("folder", None)
if not await self._metadata_manager.save_metadata(file_path, data_to_save):
# The cache still carries the repaired hash for this session; the
# next fetch retries the sidecar write.
logger.warning("Failed to persist repaired SHA256 for %s", file_path)
try:
await update_cache_func(file_path, file_path, data_to_save)
except Exception as exc: # pragma: no cover - a cache fault must not block the repair
logger.warning(
"Failed to update cache after repairing SHA256 for %s: %s",
file_path,
exc,
)
return new_hash, None
async def fetch_and_update_model(
self,
*,
@@ -301,6 +364,24 @@ class MetadataSyncService:
enable_archive = self._settings.get("enable_metadata_archive_db", False)
previous_source = model_data.get("metadata_source") or (model_data.get("civitai") or {}).get("source")
if is_empty_placeholder_hash(sha256):
# A stored hash of zero bytes (issue #1162) is not an identity: the
# provider chain would resolve it against polluted remote entries.
# Re-read the file once; refuse to look anything up when there is
# nothing to hash yet.
repaired, repair_error = await self._repair_placeholder_hash(
file_path=file_path,
model_data=model_data,
update_cache_func=update_cache_func,
)
if not repaired:
error_msg = repair_error or "Model file is empty; metadata fetch skipped"
logger.warning(
"%s (file=%s)", error_msg, os.path.basename(file_path or "")
)
return False, error_msg
sha256 = repaired
try:
provider_attempts: list[tuple[Optional[str], MetadataProviderProtocol]] = []
sqlite_attempted = False
@@ -617,6 +698,7 @@ class MetadataSyncService:
)
metadata_path = get_metadata_path(file_path)
metadata = await self._rebuild_relink_payload(file_path, metadata)
await self.update_model_metadata(
metadata_path,
metadata,
@@ -626,6 +708,50 @@ class MetadataSyncService:
return metadata
@staticmethod
async def _rebuild_relink_payload(
file_path: str, metadata: Dict[str, Any]
) -> Dict[str, Any]:
"""Return a relink payload that still carries every required local fact.
Relinking is the documented remedy after deleting a broken
``.metadata.json``, so callers frequently start from ``{}`` (or from a
half-written sidecar). Persisting the merged remote data alone would
produce a sidecar that ``BaseModelMetadata.from_dict`` cannot parse
(``file_name``/``file_path``/``size``/``modified``/``sha256`` are
required), and the scanner would then skip the model forever
(issue #1162).
Disk facts are restored first, the caller's payload is layered on top,
and the fields a relink cannot obtain from the provider (a SHA256 for a
model whose hash was lost, a display name, a preview path) are filled
from the local file.
"""
payload = await MetadataManager.load_metadata_payload(file_path)
if isinstance(metadata, dict):
payload.update(metadata)
payload.setdefault("file_path", normalize_path(file_path))
base_name = os.path.splitext(os.path.basename(file_path))[0]
if not payload.get("model_name"):
payload["model_name"] = base_name
if not payload.get("preview_url"):
payload["preview_url"] = normalize_path(
find_preview_file(base_name, get_preview_dir(file_path))
)
if not payload.get("sha256"):
try:
if os.path.isfile(file_path) and os.path.getsize(file_path) > 0:
sha256 = await calculate_sha256(file_path)
if sha256 and not is_empty_placeholder_hash(sha256):
payload["sha256"] = sha256.lower()
except OSError as exc: # pragma: no cover - defensive
logger.debug("Unable to hash %s while relinking: %s", file_path, exc)
return payload
async def _try_link_foreign_source(
self,
*,
@@ -643,6 +769,11 @@ class MetadataSyncService:
model was linked and hydrated; False leaves the caller's not-found
handling in charge.
"""
if is_empty_placeholder_hash(sha256):
# The zero-byte digest identifies no file. A repository that
# publishes a 0-byte upload "matches" it, so linking would attach
# that repository's card to an unrelated model (issue #1162).
return False
if has_external_source(model_data):
return False
try:
py/services/model_metadata_provider.py
+6
View File
@@ -4,6 +4,7 @@ import json
import logging
import random
from typing import Optional, Dict, Tuple, Any, List, Sequence
from ..utils.constants import is_empty_placeholder_hash
from .downloader import get_downloader
from .errors import RateLimitError, ResourceNotFoundError
@@ -273,6 +274,11 @@ class SQLiteModelMetadataProvider(ModelMetadataProvider):
async def get_model_by_hash(self, model_hash: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
"""Find model by hash value from SQLite database"""
if is_empty_placeholder_hash(model_hash):
# The archive database is built from remote metadata and may carry
# entries keyed by the zero-byte digest; that value identifies no
# local file, so it is never resolved (issue #1162).
return None, "Model not found"
async with self._aiosqlite.connect(self.db_path) as db:
# Look up in model_files table to get model_id and version_id
query = """
py/services/model_scanner.py
+36 -5
View File
@@ -11,6 +11,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Mapping, Optional, Sequ
from ..utils.models import BaseModelMetadata, autov3_from_civitai_files, quantization_from_civitai_files
from ..config import config
from ..utils.constants import is_empty_placeholder_hash
from ..utils.file_utils import find_preview_file, get_preview_extension, calculate_sha256, calculate_autov3
from ..utils.gc_utils import schedule_refreeze_after_rebuild
from ..utils.gguf_utils import detect_quantization
@@ -3481,6 +3482,19 @@ class ModelScanner:
if _is_pending_delete_path(file_path):
return None
# A zero-byte file is not a model. Hashing it yields the empty-string
# SHA256, which is shared by every unread/unfinished file: the library
# would show one bogus duplicate group and every member would resolve
# to the same polluted remote entry (issue #1162). Leave it unmanaged;
# a later scan picks it up once the file has content.
try:
if os.path.getsize(file_path) == 0:
logger.debug("Skipping empty model file: %s", file_path)
return None
except OSError as exc:
logger.debug("Skipping unreadable model file %s: %s", file_path, exc)
return None
metadata, should_skip = await MetadataManager.load_metadata(file_path, self.model_class)
if should_skip:
@@ -3541,8 +3555,14 @@ class ModelScanner:
if metadata is None:
metadata = await self._create_default_metadata(file_path)
assert metadata is not None
if metadata is None:
# create_default_metadata returns None for empty/vanished files and
# for failures; one bad file must never abort the whole scan.
logger.warning(
"Skipping model %s: metadata could not be created", file_path
)
return None
# Hook: allow subclasses to adjust metadata
metadata = self.adjust_metadata(metadata, file_path, root_path)
@@ -3557,11 +3577,22 @@ class ModelScanner:
# Respect hash_status='pending' (set by CheckpointScanner for large models) to defer
# hash calculation until on-demand — avoids reading entire checkpoint files at startup.
hash_status = model_data.get('hash_status', '')
if not model_data.get('sha256') and hash_status != 'pending' and file_path:
stored_sha = (model_data.get('sha256') or '').strip()
# The empty-string placeholder is not an identity: models poisoned by a
# hash of zero bytes (issue #1162) are re-hashed once the file has
# content, so an existing library heals on its next full scan.
needs_sha = not stored_sha or is_empty_placeholder_hash(stored_sha)
if needs_sha and hash_status != 'pending' and file_path:
try:
logger.info(f"Computing SHA256 hash for {file_path} (was empty from metadata)")
if stored_sha:
logger.info(
"Recomputing SHA256 for %s (stored hash is the empty-file placeholder)",
file_path,
)
else:
logger.info(f"Computing SHA256 hash for {file_path} (was empty from metadata)")
sha256 = await calculate_sha256(file_path)
if sha256:
if sha256 and not is_empty_placeholder_hash(sha256):
model_data['sha256'] = sha256.lower()
if isinstance(metadata, BaseModelMetadata):
metadata.sha256 = sha256.lower()
py/services/model_sources/modelscope.py
+16 -3
View File
@@ -46,6 +46,7 @@ import os
import re
from typing import TYPE_CHECKING, Any, Iterable, Mapping, Optional
from ...utils.constants import is_empty_placeholder_hash
from .base import (
GROUP_PREFIXES,
ModelCardContext,
@@ -206,7 +207,10 @@ class ModelScopeSource(ModelSource):
repository's weights before linking metadata to it.
"""
target = (sha256 or "").strip().lower()
if not target:
if not target or is_empty_placeholder_hash(target):
# The zero-byte digest is not a file identity: a repository that
# publishes a 0-byte upload would "prove" it owns every local model
# whose hash is that placeholder (issue #1162).
return None
data = await self._fetch_detail(source_id, cache=cache)
if data is None:
@@ -587,8 +591,17 @@ def _file_digests(data: dict[str, Any]) -> dict[str, str]:
continue
name = _clean_text(entry.get("name"))
digest = _clean_text(entry.get("sha256"))
if name and digest:
digests.setdefault(os.path.basename(name).lower(), digest.lower())
if not name or not digest:
continue
size = entry.get("size")
if isinstance(size, (int, float)) and size <= 0:
# A published 0-byte upload cannot be a model; its digest is
# the SHA256 of an empty byte string, which every unread local
# file shares (issue #1162).
continue
if is_empty_placeholder_hash(digest):
continue
digests.setdefault(os.path.basename(name).lower(), digest.lower())
return digests
py/services/other_scanner.py
+11
View File
@@ -234,6 +234,17 @@ class OtherScanner(ModelScanner):
# Calculate hash
logger.info(f"Calculating hash for other model: {file_path}")
sha256 = await calculate_sha256(real_path)
if not sha256:
# Nothing was read (empty or truncated file): no identity to
# store. Keep the lazy-hash state for a later retry and never
# index the empty-string placeholder (issue #1162).
logger.warning(
"No bytes read from model %s; keeping hash pending",
file_path,
)
metadata.hash_status = "pending"
await MetadataManager.save_metadata(file_path, metadata)
return None
# Update metadata with hash
metadata.sha256 = sha256
py/utils/file_utils.py
+13
View File
@@ -51,6 +51,10 @@ async def calculate_sha256(file_path: str) -> str:
On Windows/macOS where ``posix_fadvise`` is not available the hint is silently
skipped.
Returns ``""`` when the file contains no bytes at all: the SHA256 of an
empty byte string is not a model identity, and persisting it would fold
every unreadable/unfinished file into one bogus identity (issue #1162).
"""
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, _calculate_sha256_sync, file_path)
@@ -59,15 +63,24 @@ async def calculate_sha256(file_path: str) -> str:
def _calculate_sha256_sync(file_path: str) -> str:
sha256_hash = hashlib.sha256()
chunk_size = _get_hash_chunk_size_bytes()
read_any = False
with open(file_path, "rb") as f:
fd = f.fileno()
for byte_block in iter(lambda: f.read(chunk_size), b""):
read_any = True
sha256_hash.update(byte_block)
# Evict pages after reading so the data doesn't linger in the kernel page
# cache — on WSL this otherwise appears as unreclaimable VmmemWSL growth.
# Guard against platforms (Windows, macOS) that lack posix_fadvise.
if hasattr(os, "posix_fadvise") and hasattr(os, "POSIX_FADV_DONTNEED"):
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
if not read_any:
# A zero-byte file (or a file that was still being created/truncated
# when the scan reached it) hashes to the empty-string digest. That
# value is shared by every such file, so it can never identify a
# model; report "no hash" instead of a poisonous identity.
logger.debug("No bytes read; skipping SHA256 for empty file %s", file_path)
return ""
return sha256_hash.hexdigest()
py/utils/metadata_manager.py
+9
View File
@@ -264,6 +264,15 @@ class MetadataManager:
start_hash_time = time.perf_counter()
logger.debug(f"Calculating SHA256 hash for {real_path}...")
sha256 = await calculate_sha256(real_path)
if not sha256:
# An empty file has no model identity: writing a sidecar for it
# would persist the empty-string SHA256 placeholder and pull the
# file into the library as a bogus duplicate of every other
# unread file (issue #1162). Leave it unmanaged instead.
logger.warning(
f"Skipping metadata creation for empty model file: {file_path}"
)
return None
hash_duration = time.perf_counter() - start_hash_time
logger.info(f"SHA256 hash calculated for {real_path} in {hash_duration:.3f}s")
tests/services/test_checkpoint_lazy_hash.py
+35
View File
@@ -465,3 +465,38 @@ async def test_lora_scanner_not_affected(tmp_path: Path, monkeypatch):
# LoraScanner should use parent's _create_default_metadata which calculates hash
# We verify this by checking that it doesn't override the method
assert scanner._create_default_metadata.__qualname__ == "ModelScanner._create_default_metadata"
@pytest.mark.asyncio
async def test_calculate_hash_keeps_pending_for_an_empty_file(
tmp_path: Path, monkeypatch
):
"""An empty file yields no hash: stay pending instead of storing the
empty-string placeholder and indexing it (issue #1162)."""
checkpoints_root = tmp_path / "checkpoints"
checkpoints_root.mkdir()
checkpoint_file = checkpoints_root / "empty.safetensors"
checkpoint_file.write_bytes(b"")
normalized_root = _normalize(checkpoints_root)
normalized_file = _normalize(checkpoint_file)
monkeypatch.setattr(
model_scanner.config,
"base_models_roots",
[normalized_root],
raising=False,
)
scanner = CheckpointScanner()
metadata = await scanner._create_default_metadata(normalized_file)
assert metadata is not None
assert metadata.hash_status == "pending"
assert await scanner.calculate_hash_for_model(normalized_file) is None
metadata_file = checkpoints_root / "empty.metadata.json"
saved_data = json.loads(metadata_file.read_text(encoding="utf-8"))
assert saved_data.get("sha256") == ""
assert saved_data.get("hash_status") == "pending"
assert scanner._hash_index.has_hash("") is False
tests/services/test_civarchive_client.py
+42
View File
@@ -415,3 +415,45 @@ async def test_get_model_version_offline_cooldown_logged_as_debug(downloader, ca
r.levelno == logging.DEBUG and "while offline" in r.getMessage()
for r in module_records
)
async def test_get_model_by_hash_ignores_the_empty_hash_placeholder(downloader):
"""The zero-byte digest resolves to a pile of unrelated files (#1162)."""
from py.utils.constants import EMPTY_HASH_SHA256
client = await CivArchiveClient.get_instance()
result, error = await client.get_model_by_hash(EMPTY_HASH_SHA256)
assert result is None
assert error == "Model not found"
assert downloader.calls == []
# Nothing was remembered for a later foreign-source link either.
assert client.pop_foreign_source_ref(EMPTY_HASH_SHA256) is None
async def test_resolve_version_from_files_refuses_the_placeholder(downloader):
"""A file-entry payload for the placeholder must not record a link target."""
from py.utils.constants import EMPTY_HASH_SHA256
client = await CivArchiveClient.get_instance()
payload = {
"files": [
{
"filename": "pam-krea2_c1-st4000.safetensors",
"url": "https://www.modelscope.ai/models/ppxjmqq/pam-krea2/files?version=1",
"source": "civision",
"model_id": 474441,
"model_version_id": 199065,
}
]
}
assert (
await client._resolve_version_from_files(
payload, expected_hash=EMPTY_HASH_SHA256
)
is None
)
assert client.pop_foreign_source_ref(EMPTY_HASH_SHA256) is None
assert downloader.calls == []
tests/services/test_metadata_sync_service.py
+176
View File
@@ -6,6 +6,7 @@ import pytest
from py.services.connectivity_guard import OFFLINE_COOLDOWN_ERROR, OFFLINE_FRIENDLY_MESSAGE
from py.services.errors import MetadataPersistError, RateLimitError
from py.utils.constants import EMPTY_HASH_SHA256
from py.services.metadata_sync_service import MetadataSyncService, _merge_ordered_unique
@@ -1154,3 +1155,178 @@ async def test_fetch_and_update_model_keeps_sqlite_last_resort_after_civarchive_
civarchive_provider.get_model_by_hash.assert_awaited_once()
sqlite_provider.get_model_by_hash.assert_awaited_once()
assert model_data["metadata_source"] == "archive_db"
# ---------------------------------------------------------------------------
# Empty-file SHA256 placeholder (issue #1162)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_fetch_rehashes_the_empty_file_placeholder_before_lookup(tmp_path):
"""A stored hash of zero bytes must be replaced before any provider runs."""
import hashlib
helpers = build_service()
helpers.default_provider.get_model_by_hash.return_value = (None, "Model not found")
model_path = tmp_path / "Emilia(RE0).safetensors"
model_path.write_bytes(b"real model bytes")
expected = hashlib.sha256(b"real model bytes").hexdigest()
model_data: Dict[str, Any] = {
"model_name": "Emilia",
"folder": "root",
"file_path": str(model_path),
"sha256": EMPTY_HASH_SHA256,
}
update_cache = AsyncMock(return_value=True)
ok, error = await helpers.service.fetch_and_update_model(
sha256=EMPTY_HASH_SHA256,
file_path=str(model_path),
model_data=model_data,
update_cache_func=update_cache,
)
assert ok is False and error is not None # not found, but the hash was repaired
assert model_data["sha256"] == expected
assert model_data["hash_status"] == "completed"
helpers.default_provider.get_model_by_hash.assert_awaited_once_with(expected)
saved_payload = helpers.metadata_manager.save_metadata.await_args.args[1]
assert saved_payload["sha256"] == expected
assert "folder" not in saved_payload
# Once for the repaired hash, once for the not-found bookkeeping.
assert update_cache.await_count == 2
assert update_cache.await_args_list[0].args[2]["sha256"] == expected
@pytest.mark.asyncio
async def test_fetch_refuses_to_look_up_an_empty_file(tmp_path):
"""Zero-byte models cannot be identified: no provider request is made."""
helpers = build_service()
model_path = tmp_path / "broken.safetensors"
model_path.write_bytes(b"")
model_data: Dict[str, Any] = {"file_path": str(model_path), "sha256": EMPTY_HASH_SHA256}
update_cache = AsyncMock(return_value=True)
ok, error = await helpers.service.fetch_and_update_model(
sha256=EMPTY_HASH_SHA256,
file_path=str(model_path),
model_data=model_data,
update_cache_func=update_cache,
)
assert ok is False
assert "empty" in (error or "").lower()
helpers.default_provider.get_model_by_hash.assert_not_awaited()
update_cache.assert_not_awaited()
assert "civitai_deleted" not in model_data
@pytest.mark.asyncio
async def test_fetch_reports_a_missing_model_file_for_the_placeholder(tmp_path):
helpers = build_service()
model_data: Dict[str, Any] = {"file_path": str(tmp_path / "gone.safetensors")}
ok, error = await helpers.service.fetch_and_update_model(
sha256=EMPTY_HASH_SHA256,
file_path=str(tmp_path / "gone.safetensors"),
model_data=model_data,
update_cache_func=AsyncMock(return_value=True),
)
assert ok is False
assert "not found" in (error or "").lower()
helpers.default_provider.get_model_by_hash.assert_not_awaited()
@pytest.mark.asyncio
async def test_foreign_source_link_skipped_for_the_empty_hash_placeholder():
"""No repository gets linked for a hash that identifies no file (#1162)."""
helpers = build_service()
linked = await helpers.service._try_link_foreign_source(
sha256=EMPTY_HASH_SHA256,
file_path="/tmp/Emilia(RE0).safetensors",
model_data={"file_path": "/tmp/Emilia(RE0).safetensors"},
update_cache_func=AsyncMock(return_value=True),
)
assert linked is False
helpers.provider_selector.assert_not_awaited()
@pytest.mark.asyncio
async def test_relink_without_a_sidecar_rebuilds_the_required_local_fields(tmp_path):
"""Relinking after a manual sidecar deletion must write a parseable sidecar.
Regression test for issue #1162: relinking from ``{}`` used to persist only
the remote fields, so the next startup logged
``LoraMetadata.__init__() missing N required positional arguments`` and the
scanner skipped the model forever.
"""
import hashlib
from py.utils.models import LoraMetadata
provider = SimpleNamespace(
get_model_by_hash=AsyncMock(),
get_model_version=AsyncMock(
return_value={
"files": [
{"primary": True, "type": "Model", "hashes": {"SHA256": "ABCDEF"}}
],
"model": {"name": "Remote"},
"images": [],
}
),
)
helpers = build_service(default_provider=provider)
model_path = tmp_path / "Emilia(RE0).safetensors"
content = b"real model bytes"
model_path.write_bytes(content)
result = await helpers.service.relink_metadata(
file_path=str(model_path),
metadata={},
model_id=1,
model_version_id=2,
)
parsed = LoraMetadata.from_dict(result) # raises when a field is missing
assert parsed.file_name == "Emilia(RE0)"
assert parsed.file_path.endswith("Emilia(RE0).safetensors")
assert parsed.size == len(content)
assert parsed.sha256 == hashlib.sha256(content).hexdigest()
assert parsed.model_name == "Remote"
assert parsed.preview_url == ""
@pytest.mark.asyncio
async def test_relink_keeps_the_existing_payload_values(tmp_path):
"""A healthy sidecar is not rewritten from disk facts alone."""
provider = SimpleNamespace(
get_model_by_hash=AsyncMock(),
get_model_version=AsyncMock(
return_value={
"files": [
{"primary": True, "type": "Model", "hashes": {"SHA256": "ABCDEF"}}
],
"model": {"name": "Remote"},
"images": [],
}
),
)
helpers = build_service(default_provider=provider)
model_path = tmp_path / "model.safetensors"
model_path.write_bytes(b"real model bytes")
result = await helpers.service.relink_metadata(
file_path=str(model_path),
metadata={"file_path": str(model_path), "model_name": "Local", "sha256": "original"},
model_id=1,
model_version_id=2,
)
assert result["sha256"] == "original"
assert result["model_name"] == "Remote"
tests/services/test_model_metadata_provider.py
+17
View File
@@ -266,3 +266,20 @@ async def test_rate_limit_retrying_provider_delegates_get_version_file_mini():
assert result == {"fileName": "raw.safetensors"}
assert inner.calls == [(10, 20)]
@pytest.mark.asyncio
async def test_sqlite_provider_ignores_the_empty_hash_placeholder(tmp_path):
"""The archive DB must not resolve the zero-byte digest (#1162)."""
from py.services.model_metadata_provider import SQLiteModelMetadataProvider
from py.utils.constants import EMPTY_HASH_SHA256
db_path = tmp_path / "civitai.db"
provider = SQLiteModelMetadataProvider(str(db_path))
result, error = await provider.get_model_by_hash(EMPTY_HASH_SHA256)
assert result is None
assert error == "Model not found"
# The guard runs before the connection is opened.
assert not db_path.exists()
tests/services/test_model_scanner.py
+97 -1
View File
@@ -33,7 +33,8 @@ from py.services.pending_delete_service import (
)
from py.services.persistent_model_cache import PersistentModelCache, DEFAULT_LICENSE_FLAGS
from py.utils.civitai_utils import build_license_flags
from py.utils.models import BaseModelMetadata
from py.utils.models import BaseModelMetadata, LoraMetadata
from py.utils.sidecar_paths import get_metadata_path
class RecordingWebSocketManager:
@@ -3625,3 +3626,98 @@ async def test_on_library_changed_schedules_refreeze_after_reinit(tmp_path: Path
await asyncio.sleep(0)
assert calls == [("init", True), ("freeze", "dummy library change")]
class PluginScanner(ModelScanner):
"""Keeps the production ``_process_model_file`` (DummyScanner overrides it)."""
def __init__(self, root: Path):
self._root = str(root)
super().__init__(
model_type="dummy",
model_class=LoraMetadata,
file_extensions={".safetensors"},
hash_index=ModelHashIndex(),
)
def get_model_roots(self) -> List[str]:
return [self._root]
@pytest.mark.asyncio
async def test_process_model_file_skips_zero_byte_files(tmp_path: Path):
"""An empty model file must stay unmanaged: no sidecar, no cache entry.
Regression test for issue #1162 — hashing it would store the empty-string
SHA256 placeholder, which every unread file shares.
"""
scanner = PluginScanner(tmp_path)
empty_file = tmp_path / "Emilia(RE0).safetensors"
empty_file.write_bytes(b"")
assert await scanner._process_model_file(str(empty_file), str(tmp_path)) is None
assert not Path(get_metadata_path(str(empty_file))).exists()
@pytest.mark.asyncio
async def test_process_model_file_still_accepts_non_empty_files(tmp_path: Path):
"""Guard against over-skipping: a file with content is processed normally."""
scanner = PluginScanner(tmp_path)
model_file = tmp_path / "model.safetensors"
model_file.write_bytes(b"x" * 64)
result = await scanner._process_model_file(str(model_file), str(tmp_path))
assert result is not None
assert len(result["sha256"]) == 64
assert Path(get_metadata_path(str(model_file))).exists()
@pytest.mark.asyncio
async def test_process_model_file_rehashes_the_empty_file_placeholder(tmp_path: Path):
"""A sidecar poisoned with the empty-string hash heals during the scan.
Regression test for issue #1162: existing libraries must recover without
the user re-clicking "Verify Hashes" on every duplicate group.
"""
import hashlib
from py.utils.constants import EMPTY_HASH_SHA256
from py.utils.metadata_manager import MetadataManager
scanner = PluginScanner(tmp_path)
model_file = tmp_path / "model.safetensors"
content = b"real model bytes"
model_file.write_bytes(content)
expected = hashlib.sha256(content).hexdigest()
metadata = await MetadataManager.create_default_metadata(str(model_file))
assert metadata is not None and metadata.sha256 == expected
sidecar = Path(get_metadata_path(str(model_file)))
payload = json.loads(sidecar.read_text(encoding="utf-8"))
payload["sha256"] = EMPTY_HASH_SHA256
sidecar.write_text(json.dumps(payload), encoding="utf-8")
result = await scanner._process_model_file(str(model_file), str(tmp_path))
assert result is not None
assert result["sha256"] == expected
assert json.loads(sidecar.read_text(encoding="utf-8"))["sha256"] == expected
@pytest.mark.asyncio
async def test_process_model_file_keeps_a_zero_byte_placeholder_on_empty_files(
tmp_path: Path,
):
"""An empty file stays unmanaged instead of being "repaired" to the placeholder."""
from py.utils.constants import EMPTY_HASH_SHA256
from py.utils.metadata_manager import MetadataManager
scanner = PluginScanner(tmp_path)
model_file = tmp_path / "empty.safetensors"
model_file.write_bytes(b"")
sidecar = Path(get_metadata_path(str(model_file)))
sidecar.write_text(json.dumps({"sha256": EMPTY_HASH_SHA256}), encoding="utf-8")
assert await scanner._process_model_file(str(model_file), str(tmp_path)) is None
assert json.loads(sidecar.read_text(encoding="utf-8"))["sha256"] == EMPTY_HASH_SHA256
tests/services/test_modelscope_hydration.py
+47
View File
@@ -364,6 +364,53 @@ class TestFetchCardContextByHash:
assert await source.fetch_card_context_by_hash("owner/repo", "f3a5370b") is None
@pytest.mark.asyncio
async def test_returns_none_for_the_empty_hash_placeholder(self, monkeypatch):
"""A repository's 0-byte upload must not capture unrelated models (#1162)."""
from py.utils.constants import EMPTY_HASH_SHA256
detail = {
"ModelInfos": {
"safetensor": {
"files": [
{
"name": "pam-krea2_c1-st4000.safetensors",
"sha256": EMPTY_HASH_SHA256,
"size": 0,
}
]
}
}
}
source = ModelScopeSource()
monkeypatch.setattr(source, "_fetch_detail", _async(detail))
assert (
await source.fetch_card_context_by_hash("owner/repo", EMPTY_HASH_SHA256)
is None
)
def test_file_digests_skip_zero_byte_uploads(self):
from py.services.model_sources.modelscope import _file_digests
from py.utils.constants import EMPTY_HASH_SHA256
detail = {
"ModelInfos": {
"safetensor": {
"files": [
{"name": "real.safetensors", "sha256": "a" * 64, "size": 1024},
{
"name": "empty.safetensors",
"sha256": EMPTY_HASH_SHA256,
"size": 0,
},
]
}
}
}
assert _file_digests(detail) == {"real.safetensors": "a" * 64}
class TestSharedSourceCache:
def test_same_repository_reuses_one_memo(self):
tests/utils/test_file_utils.py
+9
View File
@@ -34,6 +34,15 @@ async def test_calculate_sha256(tmp_path):
assert result == expected_hash
@pytest.mark.asyncio
async def test_calculate_sha256_returns_blank_for_an_empty_file(tmp_path):
"""The empty-string SHA256 must never be reported as a model hash (#1162)."""
file_path = tmp_path / "empty.safetensors"
file_path.write_bytes(b"")
assert await calculate_sha256(str(file_path)) == ""
def test_find_preview_file_returns_normalized_path(tmp_path):
file_path = tmp_path / "model.preview.png"
file_path.write_bytes(b"")
tests/utils/test_metadata_manager.py
+16
View File
@@ -255,3 +255,19 @@ async def test_provenance_fields_survive_a_load_save_round_trip(tmp_path) -> Non
saved = json.loads(metadata_path.read_text(encoding="utf-8"))
assert saved["llm_confidence"] == "medium"
assert saved["llm_enriched_at"] == "2026-01-01T00:00:00+00:00"
@pytest.mark.asyncio
async def test_create_default_metadata_ignores_an_empty_model_file(tmp_path) -> None:
"""A zero-byte model must not get a sidecar carrying the empty-string hash.
Regression test for issue #1162: every unread/unfinished file hashes to the
same placeholder, so accepting it turns the library into one bogus
duplicate group and resolves it against polluted remote entries.
"""
model_path = tmp_path / "Emilia(RE0).safetensors"
model_path.write_bytes(b"")
assert await MetadataManager.create_default_metadata(str(model_path)) is None
assert not (tmp_path / "Emilia(RE0).safetensors.metadata.json").exists()