From 0531f4e9b2f20b7980e5f75ffeabb2644d3f4eaf Mon Sep 17 00:00:00 2001 From: Dariusz L Date: Sun, 2 Aug 2026 22:00:51 +0200 Subject: [PATCH] feat: expose CivArchive provider status --- core/services/search_orchestrator.py | 4 +++ core/services/search_providers.py | 15 +++++++++ core/sources/civarchive.py | 39 +++++++++++++++++++++++ tests/test_civarchive_failure_contract.py | 20 ++++++++++++ tests/test_search_service_components.py | 38 ++++++++++++++++++++++ 5 files changed, 116 insertions(+) diff --git a/core/services/search_orchestrator.py b/core/services/search_orchestrator.py index 75ab854..63562fe 100644 --- a/core/services/search_orchestrator.py +++ b/core/services/search_orchestrator.py @@ -318,6 +318,7 @@ class SearchOrchestrator: "found": False, "searched_sources": sorted(normalized_sources), "source_errors": {}, + "source_status": {}, } def iter_result_items(result): @@ -423,6 +424,8 @@ class SearchOrchestrator: for source_key, source_result in source_results.items(): if source_key == "source_errors": results["source_errors"].update(source_result or {}) + elif source_key == "source_status": + results["source_status"].update(source_result or {}) elif source_result: results[source_key] = source_result if source_found: @@ -483,6 +486,7 @@ class SearchOrchestrator: else [] ), "source_errors": {}, + "source_status": {}, } ) diff --git a/core/services/search_providers.py b/core/services/search_providers.py index 208f19b..1113e9d 100644 --- a/core/services/search_providers.py +++ b/core/services/search_providers.py @@ -523,6 +523,20 @@ class SearchProviderRunner: def handle_civarchive_error(error): error_message = f"CivArchive search failed: {error}" + status_factory = getattr(error, "as_status", None) + if callable(status_factory): + source_status = status_factory() + else: + source_status = { + "state": "unavailable", + "code": "provider_unavailable", + "retryable": True, + "http_status": None, + "message": ( + "CivArchive may be overloaded or temporarily unavailable. " + "Please try again." + ), + } self.owner.logger.warning(error_message) self.owner.search_tracker.update( request.progress_id, @@ -536,6 +550,7 @@ class SearchProviderRunner: { "civarchive": None, "source_errors": {"civarchive": error_message}, + "source_status": {"civarchive": source_status}, }, False, ) diff --git a/core/sources/civarchive.py b/core/sources/civarchive.py index e882648..c82a92a 100644 --- a/core/sources/civarchive.py +++ b/core/sources/civarchive.py @@ -66,6 +66,23 @@ DEFAULT_CIVARCHIVE_CANDIDATE_LIMIT = 10 MAX_CIVARCHIVE_CANDIDATE_LIMIT = 30 SEARCH_RESULT_DETAIL_LIMIT = 5 HASH_PAGE_MODEL_LINK_LIMIT = 5 +CIVARCHIVE_FAILURE_MESSAGES = { + "timeout": ( + "CivArchive did not respond in time. It may be temporarily overloaded. " + "Please try again." + ), + "network_error": ( + "CivArchive could not be reached. It may be temporarily unavailable. " + "Please try again." + ), + "provider_unavailable": ( + "CivArchive may be overloaded or temporarily unavailable. " + "Please try again." + ), + "rate_limited": "CivArchive rate limit was reached. Please try again later.", + "not_found": "CivArchive did not find a matching page.", + "provider_rejected": "CivArchive rejected the search request.", +} CIVARCHIVE_SIZE_PROBE_DOMAINS = ( "huggingface.co", "civarchive.com", @@ -111,6 +128,28 @@ class CivArchiveSearchError(Exception): self.http_status = http_status self.retryable = retryable + def as_status(self) -> Dict[str, Any]: + """Return a safe provider status payload for search responses.""" + if self.code == "rate_limited": + state = "rate_limited" + elif self.code == "not_found": + state = "not_found" + elif self.retryable: + state = "unavailable" + else: + state = "error" + + return { + "state": state, + "code": self.code, + "retryable": self.retryable, + "http_status": self.http_status, + "message": CIVARCHIVE_FAILURE_MESSAGES.get( + self.code, + "CivArchive search failed. Please try again later.", + ), + } + def _classify_civarchive_http_status(status_code: int) -> Dict[str, Any]: """Classify an HTTP response for user-facing provider status handling.""" diff --git a/tests/test_civarchive_failure_contract.py b/tests/test_civarchive_failure_contract.py index 7de8ea7..8b65bcf 100644 --- a/tests/test_civarchive_failure_contract.py +++ b/tests/test_civarchive_failure_contract.py @@ -75,6 +75,26 @@ def test_civarchive_search_classifies_transport_failures( assert raised.value.retryable is True +def test_civarchive_failure_exposes_safe_status_payload(): + error = civarchive.CivArchiveSearchError( + "HTTP 522", + code="provider_unavailable", + http_status=522, + retryable=True, + ) + + assert error.as_status() == { + "state": "unavailable", + "code": "provider_unavailable", + "retryable": True, + "http_status": 522, + "message": ( + "CivArchive may be overloaded or temporarily unavailable. " + "Please try again." + ), + } + + def test_request_source_response_can_propagate_transport_errors(): with patch.object(network_utils.requests, "get", side_effect=requests.Timeout("timeout")), pytest.raises( requests.Timeout, match="timeout" diff --git a/tests/test_search_service_components.py b/tests/test_search_service_components.py index daea0f9..86a15a6 100644 --- a/tests/test_search_service_components.py +++ b/tests/test_search_service_components.py @@ -298,6 +298,43 @@ def test_search_provider_runner_handles_civarchive_error(): ) +def test_search_provider_runner_exposes_civarchive_status_metadata(): + from core.sources.civarchive import CivArchiveSearchError + + owner = SimpleNamespace( + CivArchiveSearchError=CivArchiveSearchError, + logger=MagicMock(), + search_tracker=MagicMock(), + format_log_fields=MagicMock(return_value="file=model.safetensors"), + log_search_result=MagicMock(), + search_civarchive_for_file=MagicMock( + side_effect=CivArchiveSearchError( + "HTTP 522", + code="provider_unavailable", + http_status=522, + retryable=True, + ) + ), + ) + owner.search_tracker.is_cancelled.return_value = False + runner = SearchProviderRunner(owner) + + result = runner.search_civarchive_source_task( + _request(is_urn=False, data={}, civarchive_candidate_limit=5) + ) + + assert result[0]["source_status"]["civarchive"] == { + "state": "unavailable", + "code": "provider_unavailable", + "retryable": True, + "http_status": 522, + "message": ( + "CivArchive may be overloaded or temporarily unavailable. " + "Please try again." + ), + } + + def test_search_provider_runner_resolves_civitai_urn(): owner = SimpleNamespace( CivArchiveSearchError=Exception, @@ -607,6 +644,7 @@ async def test_search_orchestrator_deduplicates_local_hash_matches_and_keeps_err assert response.payload["source_errors"] == { "civarchive": "provider unavailable" } + assert response.payload["source_status"] == {} assert len(response.payload["local_hash_matches"]) == 1 assert response.payload["local_hash_matches"][0]["hash_lookup_sha256"] == ( first_hash