feat: expose CivArchive provider status

This commit is contained in:
Dariusz L
2026-08-02 22:00:51 +02:00
parent ed571a8ec6
commit 0531f4e9b2
5 changed files with 116 additions and 0 deletions
+4
View File
@@ -318,6 +318,7 @@ class SearchOrchestrator:
"found": False,
"searched_sources": sorted(normalized_sources),
"source_errors": {},
"source_status": {},
}
def iter_result_items(result):
@@ -423,6 +424,8 @@ class SearchOrchestrator:
for source_key, source_result in source_results.items():
if source_key == "source_errors":
results["source_errors"].update(source_result or {})
elif source_key == "source_status":
results["source_status"].update(source_result or {})
elif source_result:
results[source_key] = source_result
if source_found:
@@ -483,6 +486,7 @@ class SearchOrchestrator:
else []
),
"source_errors": {},
"source_status": {},
}
)
+15
View File
@@ -523,6 +523,20 @@ class SearchProviderRunner:
def handle_civarchive_error(error):
error_message = f"CivArchive search failed: {error}"
status_factory = getattr(error, "as_status", None)
if callable(status_factory):
source_status = status_factory()
else:
source_status = {
"state": "unavailable",
"code": "provider_unavailable",
"retryable": True,
"http_status": None,
"message": (
"CivArchive may be overloaded or temporarily unavailable. "
"Please try again."
),
}
self.owner.logger.warning(error_message)
self.owner.search_tracker.update(
request.progress_id,
@@ -536,6 +550,7 @@ class SearchProviderRunner:
{
"civarchive": None,
"source_errors": {"civarchive": error_message},
"source_status": {"civarchive": source_status},
},
False,
)
+39
View File
@@ -66,6 +66,23 @@ DEFAULT_CIVARCHIVE_CANDIDATE_LIMIT = 10
MAX_CIVARCHIVE_CANDIDATE_LIMIT = 30
SEARCH_RESULT_DETAIL_LIMIT = 5
HASH_PAGE_MODEL_LINK_LIMIT = 5
CIVARCHIVE_FAILURE_MESSAGES = {
"timeout": (
"CivArchive did not respond in time. It may be temporarily overloaded. "
"Please try again."
),
"network_error": (
"CivArchive could not be reached. It may be temporarily unavailable. "
"Please try again."
),
"provider_unavailable": (
"CivArchive may be overloaded or temporarily unavailable. "
"Please try again."
),
"rate_limited": "CivArchive rate limit was reached. Please try again later.",
"not_found": "CivArchive did not find a matching page.",
"provider_rejected": "CivArchive rejected the search request.",
}
CIVARCHIVE_SIZE_PROBE_DOMAINS = (
"huggingface.co",
"civarchive.com",
@@ -111,6 +128,28 @@ class CivArchiveSearchError(Exception):
self.http_status = http_status
self.retryable = retryable
def as_status(self) -> Dict[str, Any]:
"""Return a safe provider status payload for search responses."""
if self.code == "rate_limited":
state = "rate_limited"
elif self.code == "not_found":
state = "not_found"
elif self.retryable:
state = "unavailable"
else:
state = "error"
return {
"state": state,
"code": self.code,
"retryable": self.retryable,
"http_status": self.http_status,
"message": CIVARCHIVE_FAILURE_MESSAGES.get(
self.code,
"CivArchive search failed. Please try again later.",
),
}
def _classify_civarchive_http_status(status_code: int) -> Dict[str, Any]:
"""Classify an HTTP response for user-facing provider status handling."""
+20
View File
@@ -75,6 +75,26 @@ def test_civarchive_search_classifies_transport_failures(
assert raised.value.retryable is True
def test_civarchive_failure_exposes_safe_status_payload():
error = civarchive.CivArchiveSearchError(
"HTTP 522",
code="provider_unavailable",
http_status=522,
retryable=True,
)
assert error.as_status() == {
"state": "unavailable",
"code": "provider_unavailable",
"retryable": True,
"http_status": 522,
"message": (
"CivArchive may be overloaded or temporarily unavailable. "
"Please try again."
),
}
def test_request_source_response_can_propagate_transport_errors():
with patch.object(network_utils.requests, "get", side_effect=requests.Timeout("timeout")), pytest.raises(
requests.Timeout, match="timeout"
+38
View File
@@ -298,6 +298,43 @@ def test_search_provider_runner_handles_civarchive_error():
)
def test_search_provider_runner_exposes_civarchive_status_metadata():
from core.sources.civarchive import CivArchiveSearchError
owner = SimpleNamespace(
CivArchiveSearchError=CivArchiveSearchError,
logger=MagicMock(),
search_tracker=MagicMock(),
format_log_fields=MagicMock(return_value="file=model.safetensors"),
log_search_result=MagicMock(),
search_civarchive_for_file=MagicMock(
side_effect=CivArchiveSearchError(
"HTTP 522",
code="provider_unavailable",
http_status=522,
retryable=True,
)
),
)
owner.search_tracker.is_cancelled.return_value = False
runner = SearchProviderRunner(owner)
result = runner.search_civarchive_source_task(
_request(is_urn=False, data={}, civarchive_candidate_limit=5)
)
assert result[0]["source_status"]["civarchive"] == {
"state": "unavailable",
"code": "provider_unavailable",
"retryable": True,
"http_status": 522,
"message": (
"CivArchive may be overloaded or temporarily unavailable. "
"Please try again."
),
}
def test_search_provider_runner_resolves_civitai_urn():
owner = SimpleNamespace(
CivArchiveSearchError=Exception,
@@ -607,6 +644,7 @@ async def test_search_orchestrator_deduplicates_local_hash_matches_and_keeps_err
assert response.payload["source_errors"] == {
"civarchive": "provider unavailable"
}
assert response.payload["source_status"] == {}
assert len(response.payload["local_hash_matches"]) == 1
assert response.payload["local_hash_matches"][0]["hash_lookup_sha256"] == (
first_hash