feat: expose CivArchive provider status
This commit is contained in:
@@ -318,6 +318,7 @@ class SearchOrchestrator:
|
||||
"found": False,
|
||||
"searched_sources": sorted(normalized_sources),
|
||||
"source_errors": {},
|
||||
"source_status": {},
|
||||
}
|
||||
|
||||
def iter_result_items(result):
|
||||
@@ -423,6 +424,8 @@ class SearchOrchestrator:
|
||||
for source_key, source_result in source_results.items():
|
||||
if source_key == "source_errors":
|
||||
results["source_errors"].update(source_result or {})
|
||||
elif source_key == "source_status":
|
||||
results["source_status"].update(source_result or {})
|
||||
elif source_result:
|
||||
results[source_key] = source_result
|
||||
if source_found:
|
||||
@@ -483,6 +486,7 @@ class SearchOrchestrator:
|
||||
else []
|
||||
),
|
||||
"source_errors": {},
|
||||
"source_status": {},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -523,6 +523,20 @@ class SearchProviderRunner:
|
||||
|
||||
def handle_civarchive_error(error):
|
||||
error_message = f"CivArchive search failed: {error}"
|
||||
status_factory = getattr(error, "as_status", None)
|
||||
if callable(status_factory):
|
||||
source_status = status_factory()
|
||||
else:
|
||||
source_status = {
|
||||
"state": "unavailable",
|
||||
"code": "provider_unavailable",
|
||||
"retryable": True,
|
||||
"http_status": None,
|
||||
"message": (
|
||||
"CivArchive may be overloaded or temporarily unavailable. "
|
||||
"Please try again."
|
||||
),
|
||||
}
|
||||
self.owner.logger.warning(error_message)
|
||||
self.owner.search_tracker.update(
|
||||
request.progress_id,
|
||||
@@ -536,6 +550,7 @@ class SearchProviderRunner:
|
||||
{
|
||||
"civarchive": None,
|
||||
"source_errors": {"civarchive": error_message},
|
||||
"source_status": {"civarchive": source_status},
|
||||
},
|
||||
False,
|
||||
)
|
||||
|
||||
@@ -66,6 +66,23 @@ DEFAULT_CIVARCHIVE_CANDIDATE_LIMIT = 10
|
||||
MAX_CIVARCHIVE_CANDIDATE_LIMIT = 30
|
||||
SEARCH_RESULT_DETAIL_LIMIT = 5
|
||||
HASH_PAGE_MODEL_LINK_LIMIT = 5
|
||||
CIVARCHIVE_FAILURE_MESSAGES = {
|
||||
"timeout": (
|
||||
"CivArchive did not respond in time. It may be temporarily overloaded. "
|
||||
"Please try again."
|
||||
),
|
||||
"network_error": (
|
||||
"CivArchive could not be reached. It may be temporarily unavailable. "
|
||||
"Please try again."
|
||||
),
|
||||
"provider_unavailable": (
|
||||
"CivArchive may be overloaded or temporarily unavailable. "
|
||||
"Please try again."
|
||||
),
|
||||
"rate_limited": "CivArchive rate limit was reached. Please try again later.",
|
||||
"not_found": "CivArchive did not find a matching page.",
|
||||
"provider_rejected": "CivArchive rejected the search request.",
|
||||
}
|
||||
CIVARCHIVE_SIZE_PROBE_DOMAINS = (
|
||||
"huggingface.co",
|
||||
"civarchive.com",
|
||||
@@ -111,6 +128,28 @@ class CivArchiveSearchError(Exception):
|
||||
self.http_status = http_status
|
||||
self.retryable = retryable
|
||||
|
||||
def as_status(self) -> Dict[str, Any]:
|
||||
"""Return a safe provider status payload for search responses."""
|
||||
if self.code == "rate_limited":
|
||||
state = "rate_limited"
|
||||
elif self.code == "not_found":
|
||||
state = "not_found"
|
||||
elif self.retryable:
|
||||
state = "unavailable"
|
||||
else:
|
||||
state = "error"
|
||||
|
||||
return {
|
||||
"state": state,
|
||||
"code": self.code,
|
||||
"retryable": self.retryable,
|
||||
"http_status": self.http_status,
|
||||
"message": CIVARCHIVE_FAILURE_MESSAGES.get(
|
||||
self.code,
|
||||
"CivArchive search failed. Please try again later.",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _classify_civarchive_http_status(status_code: int) -> Dict[str, Any]:
|
||||
"""Classify an HTTP response for user-facing provider status handling."""
|
||||
|
||||
@@ -75,6 +75,26 @@ def test_civarchive_search_classifies_transport_failures(
|
||||
assert raised.value.retryable is True
|
||||
|
||||
|
||||
def test_civarchive_failure_exposes_safe_status_payload():
|
||||
error = civarchive.CivArchiveSearchError(
|
||||
"HTTP 522",
|
||||
code="provider_unavailable",
|
||||
http_status=522,
|
||||
retryable=True,
|
||||
)
|
||||
|
||||
assert error.as_status() == {
|
||||
"state": "unavailable",
|
||||
"code": "provider_unavailable",
|
||||
"retryable": True,
|
||||
"http_status": 522,
|
||||
"message": (
|
||||
"CivArchive may be overloaded or temporarily unavailable. "
|
||||
"Please try again."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_request_source_response_can_propagate_transport_errors():
|
||||
with patch.object(network_utils.requests, "get", side_effect=requests.Timeout("timeout")), pytest.raises(
|
||||
requests.Timeout, match="timeout"
|
||||
|
||||
@@ -298,6 +298,43 @@ def test_search_provider_runner_handles_civarchive_error():
|
||||
)
|
||||
|
||||
|
||||
def test_search_provider_runner_exposes_civarchive_status_metadata():
|
||||
from core.sources.civarchive import CivArchiveSearchError
|
||||
|
||||
owner = SimpleNamespace(
|
||||
CivArchiveSearchError=CivArchiveSearchError,
|
||||
logger=MagicMock(),
|
||||
search_tracker=MagicMock(),
|
||||
format_log_fields=MagicMock(return_value="file=model.safetensors"),
|
||||
log_search_result=MagicMock(),
|
||||
search_civarchive_for_file=MagicMock(
|
||||
side_effect=CivArchiveSearchError(
|
||||
"HTTP 522",
|
||||
code="provider_unavailable",
|
||||
http_status=522,
|
||||
retryable=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
owner.search_tracker.is_cancelled.return_value = False
|
||||
runner = SearchProviderRunner(owner)
|
||||
|
||||
result = runner.search_civarchive_source_task(
|
||||
_request(is_urn=False, data={}, civarchive_candidate_limit=5)
|
||||
)
|
||||
|
||||
assert result[0]["source_status"]["civarchive"] == {
|
||||
"state": "unavailable",
|
||||
"code": "provider_unavailable",
|
||||
"retryable": True,
|
||||
"http_status": 522,
|
||||
"message": (
|
||||
"CivArchive may be overloaded or temporarily unavailable. "
|
||||
"Please try again."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_search_provider_runner_resolves_civitai_urn():
|
||||
owner = SimpleNamespace(
|
||||
CivArchiveSearchError=Exception,
|
||||
@@ -607,6 +644,7 @@ async def test_search_orchestrator_deduplicates_local_hash_matches_and_keeps_err
|
||||
assert response.payload["source_errors"] == {
|
||||
"civarchive": "provider unavailable"
|
||||
}
|
||||
assert response.payload["source_status"] == {}
|
||||
assert len(response.payload["local_hash_matches"]) == 1
|
||||
assert response.payload["local_hash_matches"][0]["hash_lookup_sha256"] == (
|
||||
first_hash
|
||||
|
||||
Reference in New Issue
Block a user