Improve cloud API retries with transient-error classification, interruptible backoff, and DashScope-safe Retry-After handling.
This commit is contained in:
kantan-kanto
@@ -45,6 +45,13 @@ All notable changes to ComfyUI-MultiModal-Prompt-Nodes will be documented in thi
|
||||
- Passes the resolved language directly through Wan T2V/I2V API calls and Chinese correction passes instead of re-detecting modified prompts
|
||||
- Preserves direct-call compatibility by defaulting Wan's low-level T2V/I2V API helpers to shared automatic language detection
|
||||
|
||||
- Improved cloud API retry behavior for Qwen and Wan prompt rewriting
|
||||
- Retries only transient connection failures and HTTP 408, 425, 429, 500, 502, 503, and 504 responses
|
||||
- Fails immediately for authentication, request-validation, response-parsing, and other non-transient errors
|
||||
- Adds interruptible exponential backoff with jitter and `Retry-After` support
|
||||
- Retries each API request independently so a correction failure does not repeat a successful enhancement request
|
||||
- Caps the existing `max_retries` input at 10 total attempts per API operation and reports sanitized error metadata without response bodies
|
||||
|
||||
|
||||
## [1.0.14] - 2026-06-16
|
||||
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Safe retry helpers for transient cloud API failures."""
|
||||
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
|
||||
MAX_API_ATTEMPTS = 10
|
||||
RETRYABLE_STATUS_CODES = {408, 425, 429, 500, 502, 503, 504}
|
||||
_TRANSIENT_EXCEPTION_NAMES = {
|
||||
"ConnectTimeout",
|
||||
"ConnectionError",
|
||||
"ConnectionResetError",
|
||||
"ProtocolError",
|
||||
"ReadTimeout",
|
||||
"Timeout",
|
||||
"TimeoutError",
|
||||
}
|
||||
|
||||
|
||||
class APIRequestError(RuntimeError):
|
||||
"""A sanitized cloud API failure with retry classification metadata."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
operation: str,
|
||||
status_code: Optional[int] = None,
|
||||
code: Optional[str] = None,
|
||||
message: Optional[str] = None,
|
||||
request_id: Optional[str] = None,
|
||||
retry_after: Optional[float] = None,
|
||||
) -> None:
|
||||
self.operation = operation
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.service_message = message
|
||||
self.request_id = request_id
|
||||
self.retry_after = retry_after
|
||||
|
||||
details = []
|
||||
if status_code is not None:
|
||||
details.append(f"status={status_code}")
|
||||
if code:
|
||||
details.append(f"code={code}")
|
||||
if request_id:
|
||||
details.append(f"request_id={request_id}")
|
||||
suffix = f" ({', '.join(details)})" if details else ""
|
||||
super().__init__(f"{operation} failed{suffix}")
|
||||
|
||||
|
||||
def _optional_text(value: Any) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _optional_status(value: Any) -> Optional[int]:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _optional_delay(value: Any) -> Optional[float]:
|
||||
try:
|
||||
delay = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not math.isfinite(delay):
|
||||
return None
|
||||
return max(0.0, delay)
|
||||
|
||||
|
||||
def _response_value(
|
||||
response: Any,
|
||||
name: str,
|
||||
default: Any = None,
|
||||
) -> Any:
|
||||
"""Read mapping-style and object-style API responses safely."""
|
||||
if isinstance(response, Mapping):
|
||||
return response.get(name, default)
|
||||
try:
|
||||
return getattr(response, name, default)
|
||||
except (AttributeError, KeyError):
|
||||
return default
|
||||
|
||||
|
||||
def _header_value(headers: Any, name: str) -> Any:
|
||||
if isinstance(headers, Mapping):
|
||||
for key, value in headers.items():
|
||||
if str(key).lower() == name.lower():
|
||||
return value
|
||||
return None
|
||||
|
||||
try:
|
||||
getter = getattr(headers, "get")
|
||||
except (AttributeError, KeyError):
|
||||
return None
|
||||
|
||||
for key in (name, name.lower()):
|
||||
try:
|
||||
value = getter(key)
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
continue
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def api_error_from_response(response: Any, operation: str) -> APIRequestError:
|
||||
"""Build a sanitized error from a DashScope-style response object."""
|
||||
headers = _response_value(response, "headers")
|
||||
retry_after = _optional_delay(
|
||||
_response_value(response, "retry_after")
|
||||
)
|
||||
if retry_after is None:
|
||||
retry_after = _optional_delay(
|
||||
_header_value(headers, "Retry-After")
|
||||
)
|
||||
|
||||
return APIRequestError(
|
||||
operation=operation,
|
||||
status_code=_optional_status(
|
||||
_response_value(response, "status_code")
|
||||
),
|
||||
code=_optional_text(_response_value(response, "code")),
|
||||
message=_optional_text(_response_value(response, "message")),
|
||||
request_id=_optional_text(
|
||||
_response_value(response, "request_id")
|
||||
),
|
||||
retry_after=retry_after,
|
||||
)
|
||||
|
||||
|
||||
def is_retryable_api_error(error: Exception) -> bool:
|
||||
"""Return whether an error is likely to succeed on a later attempt."""
|
||||
if isinstance(error, APIRequestError):
|
||||
return error.status_code in RETRYABLE_STATUS_CODES
|
||||
if isinstance(error, (TimeoutError, ConnectionError)):
|
||||
return True
|
||||
|
||||
current: Optional[BaseException] = error
|
||||
visited = set()
|
||||
while current is not None and id(current) not in visited:
|
||||
visited.add(id(current))
|
||||
if type(current).__name__ in _TRANSIENT_EXCEPTION_NAMES:
|
||||
return True
|
||||
current = current.__cause__ or current.__context__
|
||||
return False
|
||||
|
||||
|
||||
def throw_if_processing_interrupted() -> None:
|
||||
"""Raise ComfyUI's interruption exception when processing was cancelled."""
|
||||
try:
|
||||
import comfy.model_management # type: ignore
|
||||
except ImportError:
|
||||
return
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
|
||||
def is_processing_interrupt(error: Exception) -> bool:
|
||||
try:
|
||||
import comfy.model_management # type: ignore
|
||||
return isinstance(
|
||||
error,
|
||||
comfy.model_management.InterruptProcessingException,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def normalize_max_attempts(
|
||||
max_attempts: int,
|
||||
logger: Callable[[str], None] = print,
|
||||
) -> int:
|
||||
try:
|
||||
attempts = int(max_attempts)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError("max_retries must be an integer") from error
|
||||
if attempts < 1:
|
||||
raise ValueError("max_retries must be at least 1")
|
||||
if attempts > MAX_API_ATTEMPTS:
|
||||
logger(
|
||||
f"[API Retry] Requested {attempts} attempts; "
|
||||
f"capping at {MAX_API_ATTEMPTS}"
|
||||
)
|
||||
return MAX_API_ATTEMPTS
|
||||
return attempts
|
||||
|
||||
|
||||
def compute_retry_delay(
|
||||
failed_attempt: int,
|
||||
retry_after: Optional[float] = None,
|
||||
random_fn: Callable[[], float] = random.random,
|
||||
max_delay: float = 30.0,
|
||||
) -> float:
|
||||
if retry_after is not None:
|
||||
return min(max(0.0, retry_after), max_delay)
|
||||
base_delay = min(2 ** max(0, failed_attempt - 1), max_delay)
|
||||
jitter = max(0.0, min(1.0, float(random_fn()))) * base_delay * 0.25
|
||||
return min(base_delay + jitter, max_delay)
|
||||
|
||||
|
||||
def _interruptible_sleep(
|
||||
delay: float,
|
||||
sleep_fn: Callable[[float], None],
|
||||
interrupt_check: Callable[[], None],
|
||||
) -> None:
|
||||
remaining = max(0.0, delay)
|
||||
while remaining > 0:
|
||||
interrupt_check()
|
||||
interval = min(0.25, remaining)
|
||||
sleep_fn(interval)
|
||||
remaining -= interval
|
||||
interrupt_check()
|
||||
|
||||
|
||||
def call_with_retry(
|
||||
operation: Callable[[], Any],
|
||||
max_attempts: int,
|
||||
operation_name: str,
|
||||
*,
|
||||
sleep_fn: Callable[[float], None] = time.sleep,
|
||||
random_fn: Callable[[], float] = random.random,
|
||||
interrupt_check: Callable[[], None] = throw_if_processing_interrupted,
|
||||
interrupt_error_check: Callable[[Exception], bool] = is_processing_interrupt,
|
||||
logger: Callable[[str], None] = print,
|
||||
) -> Any:
|
||||
"""Run one API operation, retrying only classified transient failures."""
|
||||
attempts = normalize_max_attempts(max_attempts, logger=logger)
|
||||
|
||||
for attempt in range(1, attempts + 1):
|
||||
interrupt_check()
|
||||
try:
|
||||
return operation()
|
||||
except Exception as error:
|
||||
if interrupt_error_check(error):
|
||||
raise
|
||||
if not is_retryable_api_error(error):
|
||||
raise
|
||||
if attempt >= attempts:
|
||||
raise
|
||||
|
||||
retry_after = (
|
||||
error.retry_after
|
||||
if isinstance(error, APIRequestError)
|
||||
else None
|
||||
)
|
||||
delay = compute_retry_delay(
|
||||
failed_attempt=attempt,
|
||||
retry_after=retry_after,
|
||||
random_fn=random_fn,
|
||||
)
|
||||
logger(
|
||||
f"[API Retry] {operation_name} failed with "
|
||||
f"{type(error).__name__}; retrying attempt "
|
||||
f"{attempt + 1}/{attempts} in {delay:.2f}s"
|
||||
)
|
||||
_interruptible_sleep(delay, sleep_fn, interrupt_check)
|
||||
|
||||
raise RuntimeError("API retry loop exited unexpectedly")
|
||||
+82
-57
@@ -53,6 +53,7 @@ try:
|
||||
resolve_output_language,
|
||||
)
|
||||
from .api_key_utils import resolve_api_key_path
|
||||
from .api_retry_utils import api_error_from_response, call_with_retry
|
||||
except ImportError:
|
||||
from qwen_api_models import (
|
||||
QWEN_API_MODELS,
|
||||
@@ -74,6 +75,7 @@ except ImportError:
|
||||
resolve_output_language,
|
||||
)
|
||||
from api_key_utils import resolve_api_key_path
|
||||
from api_retry_utils import api_error_from_response, call_with_retry
|
||||
|
||||
dashscope.base_http_api_url = 'https://dashscope-intl.aliyuncs.com/api/v1'
|
||||
|
||||
@@ -349,11 +351,9 @@ def api_edit(prompt, img_list, model="qwen3.7-plus", save_tokens=True, api_key=N
|
||||
if response.status_code == 200:
|
||||
return response.output.choices[0].message.content[0]['text']
|
||||
else:
|
||||
raise Exception(f'Failed to post: {response}')
|
||||
raise api_error_from_response(response, "Qwen-Image-Edit API request")
|
||||
|
||||
def polish_prompt_edit(api_key, prompt, img, model="qwen3.7-plus", max_retries=10, save_tokens=True, target_language="auto"):
|
||||
retries = 0
|
||||
|
||||
detected_language = detect_input_language(prompt)
|
||||
lang = resolve_output_language(target_language, detected_language)
|
||||
print(
|
||||
@@ -371,36 +371,55 @@ def polish_prompt_edit(api_key, prompt, img, model="qwen3.7-plus", max_retries=1
|
||||
prompt = f"[Please output in English] {prompt}"
|
||||
|
||||
prompt_text = f"{system_prompt}\n\nUser Input: {prompt}\n\nRewritten Prompt:"
|
||||
|
||||
while retries < max_retries:
|
||||
try:
|
||||
result = api_edit(prompt_text, img, model=model, save_tokens=save_tokens, api_key=api_key)
|
||||
|
||||
if isinstance(result, str):
|
||||
result = result.replace('```json', '').replace('```', '')
|
||||
result = json.loads(result)
|
||||
else:
|
||||
result = json.loads(result)
|
||||
|
||||
polished_prompt = result['Rewritten'].strip().replace("\n", " ")
|
||||
|
||||
if lang == "zh" and not is_acceptable_zh_output(polished_prompt):
|
||||
print("[Warning] Output language mismatch (expected simplified Chinese), converting output to simplified Chinese in a second pass")
|
||||
protected_text, placeholders = protect_quoted_text(polished_prompt, "QTXT")
|
||||
translated = api(
|
||||
build_force_translate_to_zh_prompt(protected_text, "Qwen-Image-Edit"),
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
)
|
||||
polished_prompt = restore_quoted_text(translated.strip().replace("\n", " "), placeholders)
|
||||
result = call_with_retry(
|
||||
lambda: api_edit(
|
||||
prompt_text,
|
||||
img,
|
||||
model=model,
|
||||
save_tokens=save_tokens,
|
||||
api_key=api_key,
|
||||
),
|
||||
max_attempts=max_retries,
|
||||
operation_name="Qwen-Image-Edit prompt generation",
|
||||
)
|
||||
|
||||
return polished_prompt
|
||||
except Exception as e:
|
||||
error = e
|
||||
retries += 1
|
||||
print(f"[Warning] Error during API call (attempt {retries}/{max_retries}): {e}")
|
||||
|
||||
raise EnvironmentError(f"Error during API call: {error}")
|
||||
try:
|
||||
if not isinstance(result, str):
|
||||
raise TypeError("response content is not text")
|
||||
parsed = json.loads(
|
||||
result.replace('```json', '').replace('```', '')
|
||||
)
|
||||
rewritten = parsed["Rewritten"]
|
||||
if not isinstance(rewritten, str):
|
||||
raise TypeError("Rewritten is not text")
|
||||
polished_prompt = rewritten.strip().replace("\n", " ")
|
||||
except (json.JSONDecodeError, KeyError, TypeError) as error:
|
||||
raise ValueError(
|
||||
"Qwen-Image-Edit returned an invalid response format"
|
||||
) from error
|
||||
|
||||
if lang == "zh" and not is_acceptable_zh_output(polished_prompt):
|
||||
print("[Warning] Output language mismatch (expected simplified Chinese), converting output to simplified Chinese in a second pass")
|
||||
protected_text, placeholders = protect_quoted_text(polished_prompt, "QTXT")
|
||||
translated = call_with_retry(
|
||||
lambda: api(
|
||||
build_force_translate_to_zh_prompt(
|
||||
protected_text,
|
||||
"Qwen-Image-Edit",
|
||||
),
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
),
|
||||
max_attempts=max_retries,
|
||||
operation_name="Qwen-Image-Edit language correction",
|
||||
)
|
||||
polished_prompt = restore_quoted_text(
|
||||
translated.strip().replace("\n", " "),
|
||||
placeholders,
|
||||
)
|
||||
|
||||
return polished_prompt
|
||||
|
||||
def api(prompt, model, api_key=None, kwargs={}):
|
||||
if not api_key:
|
||||
@@ -427,11 +446,9 @@ def api(prompt, model, api_key=None, kwargs={}):
|
||||
if response.status_code == 200:
|
||||
return response.output.choices[0].message.content
|
||||
else:
|
||||
raise Exception(f'Failed to post: {response}')
|
||||
raise api_error_from_response(response, "Qwen-Image API request")
|
||||
|
||||
def polish_prompt(api_key, prompt, model="qwen3.7-plus", max_retries=10, target_language="auto"):
|
||||
retries = 0
|
||||
|
||||
detected_language = detect_input_language(prompt)
|
||||
lang = resolve_output_language(target_language, detected_language)
|
||||
print(
|
||||
@@ -449,29 +466,37 @@ def polish_prompt(api_key, prompt, model="qwen3.7-plus", max_retries=10, target_
|
||||
magic_prompt = "超清，4K，电影级构图" if lang == 'zh' else "Ultra HD, 4K, cinematic composition"
|
||||
|
||||
prompt_text = f"{system_prompt}\n\nUser Input: {prompt}\n\nRewritten Prompt:"
|
||||
|
||||
while retries < max_retries:
|
||||
try:
|
||||
result = api(prompt_text, model=model, api_key=api_key)
|
||||
polished_prompt = result.strip().replace("\n", " ")
|
||||
|
||||
if lang == "zh" and not is_acceptable_zh_output(polished_prompt):
|
||||
print("[Warning] Output language mismatch (expected simplified Chinese), converting output to simplified Chinese in a second pass")
|
||||
protected_text, placeholders = protect_quoted_text(polished_prompt, "QTXT")
|
||||
translated = api(
|
||||
build_force_translate_to_zh_prompt(protected_text, "Qwen-Image"),
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
)
|
||||
polished_prompt = restore_quoted_text(translated.strip().replace("\n", " "), placeholders)
|
||||
result = call_with_retry(
|
||||
lambda: api(prompt_text, model=model, api_key=api_key),
|
||||
max_attempts=max_retries,
|
||||
operation_name="Qwen-Image prompt generation",
|
||||
)
|
||||
if not isinstance(result, str):
|
||||
raise ValueError("Qwen-Image returned an invalid response format")
|
||||
polished_prompt = result.strip().replace("\n", " ")
|
||||
|
||||
return append_prompt_suffix(polished_prompt, magic_prompt, lang)
|
||||
except Exception as e:
|
||||
error = e
|
||||
retries += 1
|
||||
print(f"[Warning] Error during API call (attempt {retries}/{max_retries}): {e}")
|
||||
|
||||
raise EnvironmentError(f"Error during API call: {error}")
|
||||
if lang == "zh" and not is_acceptable_zh_output(polished_prompt):
|
||||
print("[Warning] Output language mismatch (expected simplified Chinese), converting output to simplified Chinese in a second pass")
|
||||
protected_text, placeholders = protect_quoted_text(polished_prompt, "QTXT")
|
||||
translated = call_with_retry(
|
||||
lambda: api(
|
||||
build_force_translate_to_zh_prompt(
|
||||
protected_text,
|
||||
"Qwen-Image",
|
||||
),
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
),
|
||||
max_attempts=max_retries,
|
||||
operation_name="Qwen-Image language correction",
|
||||
)
|
||||
polished_prompt = restore_quoted_text(
|
||||
translated.strip().replace("\n", " "),
|
||||
placeholders,
|
||||
)
|
||||
|
||||
return append_prompt_suffix(polished_prompt, magic_prompt, lang)
|
||||
|
||||
class QwenImageEditPromptGenerator:
|
||||
@classmethod
|
||||
@@ -515,8 +540,8 @@ class QwenImageEditPromptGenerator:
|
||||
"tooltip": "mmproj file (required for Local model only, select manually or use auto-detect)"
|
||||
}),
|
||||
"max_retries": ("INT",{
|
||||
"default": 3, "min": 1, "max": 10000, "step": 1,
|
||||
"tooltip": "Maximum number of retries when an API call fails."
|
||||
"default": 3, "min": 1, "max": 10, "step": 1,
|
||||
"tooltip": "Maximum total API attempts for transient failures."
|
||||
}),
|
||||
"device": (["CPU", "GPU"], {
|
||||
"default": "CPU",
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from api_retry_utils import (
|
||||
APIRequestError,
|
||||
api_error_from_response,
|
||||
call_with_retry,
|
||||
is_retryable_api_error,
|
||||
normalize_max_attempts,
|
||||
)
|
||||
|
||||
|
||||
class KeyErrorAttributeResponse:
|
||||
"""Mimic DashScope responses that raise KeyError for missing fields."""
|
||||
|
||||
def __init__(self, **values):
|
||||
self._values = values
|
||||
|
||||
def __getattr__(self, name):
|
||||
try:
|
||||
return self._values[name]
|
||||
except KeyError:
|
||||
raise KeyError(name) from None
|
||||
|
||||
|
||||
class APIErrorTests(unittest.TestCase):
|
||||
def test_response_error_is_sanitized(self):
|
||||
response = SimpleNamespace(
|
||||
status_code=429,
|
||||
code="Throttled",
|
||||
message="secret response body",
|
||||
request_id="request-123",
|
||||
headers={"Retry-After": "4"},
|
||||
)
|
||||
|
||||
error = api_error_from_response(response, "test request")
|
||||
|
||||
self.assertEqual(error.status_code, 429)
|
||||
self.assertEqual(error.retry_after, 4.0)
|
||||
self.assertEqual(error.service_message, "secret response body")
|
||||
self.assertIn("code=Throttled", str(error))
|
||||
self.assertIn("request_id=request-123", str(error))
|
||||
self.assertNotIn("secret response body", str(error))
|
||||
|
||||
def test_dashscope_style_missing_fields_are_safe(self):
|
||||
response = KeyErrorAttributeResponse(
|
||||
status_code=429,
|
||||
code="Throttled",
|
||||
message="secret response body",
|
||||
request_id="request-123",
|
||||
)
|
||||
|
||||
error = api_error_from_response(response, "test request")
|
||||
|
||||
self.assertEqual(error.status_code, 429)
|
||||
self.assertIsNone(error.retry_after)
|
||||
self.assertTrue(is_retryable_api_error(error))
|
||||
self.assertNotIn("secret response body", str(error))
|
||||
|
||||
def test_none_retry_after_falls_back_to_header(self):
|
||||
response = SimpleNamespace(
|
||||
status_code=429,
|
||||
retry_after=None,
|
||||
headers={"Retry-After": "4"},
|
||||
)
|
||||
|
||||
error = api_error_from_response(response, "test request")
|
||||
|
||||
self.assertEqual(error.retry_after, 4.0)
|
||||
|
||||
def test_invalid_retry_after_falls_back_to_case_insensitive_header(self):
|
||||
response = SimpleNamespace(
|
||||
status_code=503,
|
||||
retry_after="not-a-delay",
|
||||
headers={"retry-after": "5"},
|
||||
)
|
||||
|
||||
error = api_error_from_response(response, "test request")
|
||||
|
||||
self.assertEqual(error.retry_after, 5.0)
|
||||
|
||||
def test_valid_retry_after_attribute_takes_precedence(self):
|
||||
response = SimpleNamespace(
|
||||
status_code=503,
|
||||
retry_after="2",
|
||||
headers={"Retry-After": "4"},
|
||||
)
|
||||
|
||||
error = api_error_from_response(response, "test request")
|
||||
|
||||
self.assertEqual(error.retry_after, 2.0)
|
||||
|
||||
def test_mapping_response_is_supported(self):
|
||||
response = {
|
||||
"status_code": 500,
|
||||
"code": "InternalError",
|
||||
"headers": {"Retry-After": "3"},
|
||||
}
|
||||
|
||||
error = api_error_from_response(response, "test request")
|
||||
|
||||
self.assertEqual(error.status_code, 500)
|
||||
self.assertEqual(error.code, "InternalError")
|
||||
self.assertEqual(error.retry_after, 3.0)
|
||||
|
||||
def test_attempt_count_is_capped(self):
|
||||
messages = []
|
||||
|
||||
self.assertEqual(normalize_max_attempts(10000, messages.append), 10)
|
||||
self.assertIn("capping at 10", messages[0])
|
||||
|
||||
|
||||
class RetryTests(unittest.TestCase):
|
||||
def _run(self, operation, max_attempts=3, **kwargs):
|
||||
sleeps = []
|
||||
messages = []
|
||||
result = call_with_retry(
|
||||
operation,
|
||||
max_attempts=max_attempts,
|
||||
operation_name="test request",
|
||||
sleep_fn=sleeps.append,
|
||||
random_fn=lambda: 0.0,
|
||||
interrupt_check=kwargs.pop("interrupt_check", lambda: None),
|
||||
interrupt_error_check=lambda error: False,
|
||||
logger=messages.append,
|
||||
**kwargs,
|
||||
)
|
||||
return result, sleeps, messages
|
||||
|
||||
def test_retries_transient_status_then_succeeds(self):
|
||||
calls = 0
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise APIRequestError("test", status_code=429)
|
||||
return "ok"
|
||||
|
||||
result, sleeps, messages = self._run(operation)
|
||||
|
||||
self.assertEqual(result, "ok")
|
||||
self.assertEqual(calls, 2)
|
||||
self.assertAlmostEqual(sum(sleeps), 1.0)
|
||||
self.assertEqual(len(messages), 1)
|
||||
|
||||
def test_retries_timeout_then_succeeds(self):
|
||||
calls = 0
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise TimeoutError("timed out")
|
||||
return "ok"
|
||||
|
||||
result, _, _ = self._run(operation)
|
||||
|
||||
self.assertEqual(result, "ok")
|
||||
self.assertEqual(calls, 2)
|
||||
|
||||
def test_honors_retry_after(self):
|
||||
calls = 0
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise APIRequestError(
|
||||
"test",
|
||||
status_code=503,
|
||||
retry_after=4.0,
|
||||
)
|
||||
return "ok"
|
||||
|
||||
_, sleeps, _ = self._run(operation)
|
||||
|
||||
self.assertAlmostEqual(sum(sleeps), 4.0)
|
||||
|
||||
def test_does_not_retry_non_transient_status(self):
|
||||
calls = 0
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise APIRequestError("test", status_code=401)
|
||||
|
||||
with self.assertRaises(APIRequestError):
|
||||
self._run(operation)
|
||||
|
||||
self.assertEqual(calls, 1)
|
||||
|
||||
def test_does_not_retry_response_parsing_error(self):
|
||||
calls = 0
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise ValueError("invalid response")
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
self._run(operation)
|
||||
|
||||
self.assertEqual(calls, 1)
|
||||
|
||||
def test_stops_after_last_attempt_without_final_sleep(self):
|
||||
calls = 0
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise APIRequestError("test", status_code=500)
|
||||
|
||||
sleeps = []
|
||||
with self.assertRaises(APIRequestError):
|
||||
call_with_retry(
|
||||
operation,
|
||||
max_attempts=3,
|
||||
operation_name="test request",
|
||||
sleep_fn=sleeps.append,
|
||||
random_fn=lambda: 0.0,
|
||||
interrupt_check=lambda: None,
|
||||
interrupt_error_check=lambda error: False,
|
||||
logger=lambda message: None,
|
||||
)
|
||||
|
||||
self.assertEqual(calls, 3)
|
||||
self.assertAlmostEqual(sum(sleeps), 3.0)
|
||||
|
||||
def test_interrupt_during_backoff_stops_retrying(self):
|
||||
calls = 0
|
||||
checks = 0
|
||||
|
||||
class Interrupted(Exception):
|
||||
pass
|
||||
|
||||
def operation():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
raise APIRequestError("test", status_code=503)
|
||||
|
||||
def interrupt_check():
|
||||
nonlocal checks
|
||||
checks += 1
|
||||
if checks == 2:
|
||||
raise Interrupted()
|
||||
|
||||
with self.assertRaises(Interrupted):
|
||||
self._run(operation, interrupt_check=interrupt_check)
|
||||
|
||||
self.assertEqual(calls, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -74,6 +74,40 @@ class WanCloudLanguageRoutingTests(unittest.TestCase):
|
||||
self.assertEqual(result, "少女缓慢地看向镜头")
|
||||
self.assertEqual(calls[0]["language"], "zh")
|
||||
|
||||
def test_correction_failure_does_not_repeat_successful_enhancement(self):
|
||||
enhancement_calls = 0
|
||||
correction_calls = 0
|
||||
|
||||
def fake_api(prompt, **kwargs):
|
||||
nonlocal enhancement_calls
|
||||
enhancement_calls += 1
|
||||
return "A girl slowly looks at the camera"
|
||||
|
||||
def fake_correction(*args, **kwargs):
|
||||
nonlocal correction_calls
|
||||
correction_calls += 1
|
||||
raise ValueError("invalid correction response")
|
||||
|
||||
with (
|
||||
mock.patch.object(wan_nodes, "api", side_effect=fake_api),
|
||||
mock.patch.object(
|
||||
wan_nodes,
|
||||
"translate_wan_output_to_zh_api",
|
||||
side_effect=fake_correction,
|
||||
),
|
||||
):
|
||||
with self.assertRaises(ValueError):
|
||||
wan_nodes.polish_prompt_wan(
|
||||
"key",
|
||||
"少女がゆっくりカメラを見る",
|
||||
task_type="t2v",
|
||||
max_retries=3,
|
||||
target_language="zh",
|
||||
)
|
||||
|
||||
self.assertEqual(enhancement_calls, 1)
|
||||
self.assertEqual(correction_calls, 1)
|
||||
|
||||
def test_kana_only_auto_routes_i2v_to_chinese(self):
|
||||
calls = []
|
||||
|
||||
|
||||
+61
-50
@@ -44,6 +44,7 @@ try:
|
||||
resolve_output_language,
|
||||
)
|
||||
from .api_key_utils import resolve_api_key_path
|
||||
from .api_retry_utils import api_error_from_response, call_with_retry
|
||||
except ImportError:
|
||||
from qwen_api_models import (
|
||||
QWEN_API_MODELS,
|
||||
@@ -62,6 +63,7 @@ except ImportError:
|
||||
resolve_output_language,
|
||||
)
|
||||
from api_key_utils import resolve_api_key_path
|
||||
from api_retry_utils import api_error_from_response, call_with_retry
|
||||
|
||||
# for configuration
|
||||
dashscope.base_http_api_url = 'https://dashscope-intl.aliyuncs.com/api/v1'
|
||||
@@ -274,7 +276,7 @@ def api_with_image(
|
||||
if response.status_code == 200:
|
||||
return response.output.choices[0].message.content[0]['text']
|
||||
else:
|
||||
raise Exception(f'Failed to post: {response}')
|
||||
raise api_error_from_response(response, "Wan I2V API request")
|
||||
|
||||
def api(
|
||||
prompt,
|
||||
@@ -318,7 +320,7 @@ def api(
|
||||
if response.status_code == 200:
|
||||
return response.output.choices[0].message.content
|
||||
else:
|
||||
raise Exception(f'Failed to post: {response}')
|
||||
raise api_error_from_response(response, "Wan T2V API request")
|
||||
|
||||
def polish_prompt_wan(api_key, prompt, task_type="t2v", model="qwen3.7-plus", max_retries=10, image=None, save_tokens=True, target_language="auto"):
|
||||
"""
|
||||
@@ -329,59 +331,68 @@ def polish_prompt_wan(api_key, prompt, task_type="t2v", model="qwen3.7-plus", ma
|
||||
prompt: Original prompt
|
||||
task_type: "t2v" for text-to-video or "i2v" for image-to-video
|
||||
model: Qwen model to use
|
||||
max_retries: Maximum retry attempts
|
||||
max_retries: Maximum total API attempts for transient failures
|
||||
image: PIL Image object (required for I2V)
|
||||
save_tokens: Whether to compress image for token saving
|
||||
"""
|
||||
detected_language = detect_input_language(prompt)
|
||||
language = resolve_output_language(target_language, detected_language)
|
||||
retries = 0
|
||||
|
||||
while retries < max_retries:
|
||||
try:
|
||||
if task_type == "i2v" and image is not None:
|
||||
# Use vision model for I2V
|
||||
result = api_with_image(
|
||||
prompt,
|
||||
image,
|
||||
model=model,
|
||||
task_type=task_type,
|
||||
save_tokens=save_tokens,
|
||||
api_key=api_key,
|
||||
language=language,
|
||||
)
|
||||
else:
|
||||
# Use text model for T2V
|
||||
result = api(
|
||||
prompt,
|
||||
model=model,
|
||||
task_type=task_type,
|
||||
api_key=api_key,
|
||||
language=language,
|
||||
)
|
||||
|
||||
polished_prompt = result.strip().replace("\n", " ")
|
||||
|
||||
if language == "zh" and not contains_cjk(polished_prompt):
|
||||
print("[Warning] Output language mismatch (expected zh), converting output to simplified Chinese in a second pass")
|
||||
protected_text, placeholders = protect_quoted_text(polished_prompt, "WTXT")
|
||||
translated = translate_wan_output_to_zh_api(
|
||||
api_key=api_key,
|
||||
text=protected_text,
|
||||
model=model,
|
||||
task_type=task_type,
|
||||
image=image,
|
||||
save_tokens=save_tokens,
|
||||
)
|
||||
return restore_quoted_text(translated.strip().replace("\n", " "), placeholders)
|
||||
if task_type == "i2v" and image is not None:
|
||||
operation = lambda: api_with_image(
|
||||
prompt,
|
||||
image,
|
||||
model=model,
|
||||
task_type=task_type,
|
||||
save_tokens=save_tokens,
|
||||
api_key=api_key,
|
||||
language=language,
|
||||
)
|
||||
operation_name = "Wan I2V prompt generation"
|
||||
else:
|
||||
operation = lambda: api(
|
||||
prompt,
|
||||
model=model,
|
||||
task_type=task_type,
|
||||
api_key=api_key,
|
||||
language=language,
|
||||
)
|
||||
operation_name = "Wan T2V prompt generation"
|
||||
|
||||
return polished_prompt
|
||||
except Exception as e:
|
||||
error = e
|
||||
retries += 1
|
||||
print(f"[Warning] Error during API call (attempt {retries}/{max_retries}): {e}")
|
||||
|
||||
raise EnvironmentError(f"Error during API call: {error}")
|
||||
result = call_with_retry(
|
||||
operation,
|
||||
max_attempts=max_retries,
|
||||
operation_name=operation_name,
|
||||
)
|
||||
if not isinstance(result, str):
|
||||
raise ValueError("Wan API returned an invalid response format")
|
||||
polished_prompt = result.strip().replace("\n", " ")
|
||||
|
||||
if language == "zh" and not contains_cjk(polished_prompt):
|
||||
print("[Warning] Output language mismatch (expected zh), converting output to simplified Chinese in a second pass")
|
||||
protected_text, placeholders = protect_quoted_text(polished_prompt, "WTXT")
|
||||
translated = call_with_retry(
|
||||
lambda: translate_wan_output_to_zh_api(
|
||||
api_key=api_key,
|
||||
text=protected_text,
|
||||
model=model,
|
||||
task_type=task_type,
|
||||
image=image,
|
||||
save_tokens=save_tokens,
|
||||
),
|
||||
max_attempts=max_retries,
|
||||
operation_name="Wan language correction",
|
||||
)
|
||||
if not isinstance(translated, str):
|
||||
raise ValueError(
|
||||
"Wan language correction returned an invalid response format"
|
||||
)
|
||||
return restore_quoted_text(
|
||||
translated.strip().replace("\n", " "),
|
||||
placeholders,
|
||||
)
|
||||
|
||||
return polished_prompt
|
||||
|
||||
class WanVideoPromptGenerator:
|
||||
@classmethod
|
||||
@@ -425,8 +436,8 @@ class WanVideoPromptGenerator:
|
||||
"tooltip": "mmproj file (required for Local model, select manually or use auto-detect)"
|
||||
}),
|
||||
"max_retries": ("INT", {
|
||||
"default": 3, "min": 1, "max": 10000, "step": 1,
|
||||
"tooltip": "Maximum number of retries when an API call fails"
|
||||
"default": 3, "min": 1, "max": 10, "step": 1,
|
||||
"tooltip": "Maximum total API attempts for transient failures"
|
||||
}),
|
||||
"device": (["CPU", "GPU"], {
|
||||
"default": "CPU",
|
||||
|
||||
Reference in New Issue
Block a user