feat: add LoggingUtils class with base64 string truncation
- Prevent console spam from base64 image data in API payloads - Recursively truncate base64 strings in dictionaries and lists - Show first 50 chars + truncation message for debugging clarity
This commit is contained in:
@@ -3,7 +3,7 @@ import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from .byteplus_utils import BytePlusConfig, BytePlusImageUtils
|
||||
from .byteplus_utils import BytePlusConfig, BytePlusImageUtils, LoggingUtils
|
||||
|
||||
|
||||
class SeedResultProcessor:
|
||||
@@ -67,7 +67,7 @@ class SeedImageApiHandler:
|
||||
print(f"Making Seed image request to: {config.get_base_url()}/images/generations")
|
||||
print(f"Model: {model}")
|
||||
print(f"Requested batch size (n): {payload.get('n', 1)}")
|
||||
print(f"Payload: {payload}")
|
||||
LoggingUtils.safe_log_payload(payload, "Payload")
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
@@ -79,7 +79,7 @@ class SeedImageApiHandler:
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"Seed image generation successful: {result}")
|
||||
LoggingUtils.safe_log_payload(result, "Seed image generation successful")
|
||||
return result
|
||||
else:
|
||||
print(f"Seed image generation error: {response.status_code} - {response.text}")
|
||||
|
||||
+38
-3
@@ -6,6 +6,7 @@ import os
|
||||
import time
|
||||
import tempfile
|
||||
from typing import Optional, Dict, Any
|
||||
import copy
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
@@ -13,6 +14,40 @@ import torch
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class LoggingUtils:
|
||||
"""Utility functions for safe logging that truncates base64 strings."""
|
||||
|
||||
@staticmethod
|
||||
def truncate_base64_in_dict(data, max_length=50):
|
||||
"""Recursively truncate base64 strings in a dictionary for logging."""
|
||||
if isinstance(data, dict):
|
||||
truncated = {}
|
||||
for key, value in data.items():
|
||||
truncated[key] = LoggingUtils.truncate_base64_in_dict(value, max_length)
|
||||
return truncated
|
||||
elif isinstance(data, list):
|
||||
return [LoggingUtils.truncate_base64_in_dict(item, max_length) for item in data]
|
||||
elif isinstance(data, str):
|
||||
# Check if string looks like base64 data URI
|
||||
if data.startswith('data:image/') and 'base64,' in data and len(data) > 100:
|
||||
# Find the base64 part
|
||||
base64_start = data.find('base64,') + 7
|
||||
prefix = data[:base64_start]
|
||||
base64_part = data[base64_start:]
|
||||
if len(base64_part) > max_length:
|
||||
truncated_b64 = base64_part[:max_length] + f"...[{len(base64_part)-max_length} chars truncated]"
|
||||
return prefix + truncated_b64
|
||||
return data
|
||||
else:
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def safe_log_payload(payload, description="Payload"):
|
||||
"""Safely log a payload by truncating base64 strings."""
|
||||
safe_payload = LoggingUtils.truncate_base64_in_dict(copy.deepcopy(payload))
|
||||
print(f"{description}: {safe_payload}")
|
||||
|
||||
|
||||
class BytePlusConfig:
|
||||
"""Singleton class to handle BytePlus configuration and client setup."""
|
||||
|
||||
@@ -141,7 +176,7 @@ class BytePlusApiHandler:
|
||||
print(f"API Key length: {len(config.get_api_key()) if config.get_api_key() else 0}")
|
||||
print(f"API Key starts with: {config.get_api_key()[:10] if config.get_api_key() else 'None'}...")
|
||||
print(f"Headers: {headers}")
|
||||
print(f"Payload: {payload}")
|
||||
LoggingUtils.safe_log_payload(payload, "Payload")
|
||||
|
||||
if callback_url:
|
||||
payload["callback_url"] = callback_url
|
||||
@@ -208,8 +243,8 @@ class BytePlusApiHandler:
|
||||
print(f"Task {task_id} status: {status}")
|
||||
|
||||
if status == "succeeded":
|
||||
# Debug: Print the full response to understand the structure
|
||||
print(f"Full successful response: {result}")
|
||||
# Debug: Print the full response to understand the structure (with base64 truncation)
|
||||
LoggingUtils.safe_log_payload(result, "Full successful response")
|
||||
|
||||
# Look for video URL in the result - try multiple possible paths
|
||||
if "content" in result and "video_url" in result["content"]:
|
||||
|
||||
@@ -3,6 +3,7 @@ import torch
|
||||
import requests
|
||||
import tempfile
|
||||
import numpy as np
|
||||
import time
|
||||
from PIL import Image
|
||||
|
||||
# Try to import opencv - if not available, provide error message
|
||||
@@ -15,9 +16,97 @@ except ImportError:
|
||||
print("To install: pip install opencv-python>=4.5.0")
|
||||
|
||||
|
||||
def _fetch_video(url, stream=True):
|
||||
"""Download video content from URL."""
|
||||
return requests.get(url, stream=stream).content
|
||||
|
||||
def _fetch_video(url, timeout=300):
|
||||
"""Download video content from URL with streaming and proper total timeout enforcement."""
|
||||
try:
|
||||
print(f"Starting video download from: {url}")
|
||||
print(f"Total timeout: {timeout} seconds")
|
||||
|
||||
# Record start time for total timeout enforcement
|
||||
start_time = time.time()
|
||||
|
||||
# Set headers to mimic a browser request
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'Accept': 'video/*, */*',
|
||||
'Accept-Encoding': 'identity', # Disable compression to avoid issues
|
||||
'Connection': 'keep-alive'
|
||||
}
|
||||
|
||||
# Start streaming request with connection timeout only
|
||||
# We'll handle total timeout manually
|
||||
try:
|
||||
response = requests.get(url, stream=True, timeout=30, headers=headers) # 30s connection timeout only
|
||||
response.raise_for_status() # Raise exception for bad status codes
|
||||
except requests.exceptions.Timeout:
|
||||
raise Exception("Connection timeout - could not connect to server within 30 seconds")
|
||||
|
||||
# Check if we've already exceeded timeout during connection
|
||||
elapsed = time.time() - start_time
|
||||
if elapsed >= timeout:
|
||||
response.close()
|
||||
raise Exception(f"Total timeout exceeded during connection ({elapsed:.1f}s >= {timeout}s)")
|
||||
|
||||
# Get file size if available
|
||||
total_size = response.headers.get('content-length')
|
||||
if total_size:
|
||||
total_size = int(total_size)
|
||||
print(f"Video size: {total_size / (1024*1024):.2f} MB")
|
||||
else:
|
||||
print("Video size: Unknown")
|
||||
|
||||
print(f"Starting download, will timeout after {timeout} seconds...")
|
||||
|
||||
# Download in chunks with strict total timeout enforcement
|
||||
video_content = bytearray()
|
||||
downloaded = 0
|
||||
chunk_size = 1024 * 1024 # 1MB chunks
|
||||
last_progress_logged = 0
|
||||
last_timeout_check = start_time
|
||||
|
||||
try:
|
||||
for chunk in response.iter_content(chunk_size=chunk_size, decode_unicode=False):
|
||||
# Check total timeout every few chunks (not every single chunk for performance)
|
||||
current_time = time.time()
|
||||
if current_time - last_timeout_check > 5: # Check every 5 seconds
|
||||
elapsed = current_time - start_time
|
||||
if elapsed >= timeout:
|
||||
response.close()
|
||||
raise Exception(f"Download timeout exceeded: {elapsed:.1f}s >= {timeout}s (downloaded {downloaded/(1024*1024):.2f} MB)")
|
||||
last_timeout_check = current_time
|
||||
|
||||
if chunk:
|
||||
video_content.extend(chunk)
|
||||
downloaded += len(chunk)
|
||||
|
||||
# Log progress every 10% or every 10MB, whichever comes first
|
||||
if total_size:
|
||||
progress = (downloaded / total_size) * 100
|
||||
if progress - last_progress_logged >= 10:
|
||||
elapsed = time.time() - start_time
|
||||
print(f"Download progress: {progress:.1f}% ({downloaded / (1024*1024):.2f} MB) - {elapsed:.1f}s elapsed")
|
||||
last_progress_logged = progress
|
||||
else:
|
||||
# Log every 10MB when size is unknown
|
||||
mb_downloaded = downloaded / (1024*1024)
|
||||
if mb_downloaded - last_progress_logged >= 10:
|
||||
elapsed = time.time() - start_time
|
||||
print(f"Downloaded: {mb_downloaded:.2f} MB - {elapsed:.1f}s elapsed")
|
||||
last_progress_logged = mb_downloaded
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
final_elapsed = time.time() - start_time
|
||||
print(f"Download complete: {len(video_content)} bytes in {final_elapsed:.1f} seconds")
|
||||
return bytes(video_content)
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
raise Exception(f"Download timeout after {timeout}s - video may be too large or connection too slow")
|
||||
except requests.exceptions.RequestException as e:
|
||||
raise Exception(f"Download failed: {str(e)}")
|
||||
except Exception as e:
|
||||
raise Exception(f"Unexpected error during download: {str(e)}")
|
||||
|
||||
|
||||
class VideoToFrames:
|
||||
@@ -62,14 +151,18 @@ class VideoToFrames:
|
||||
frame_count = 0
|
||||
video_fps = 0.0
|
||||
|
||||
# Download video content
|
||||
# Download video content (5 minute timeout)
|
||||
try:
|
||||
print(f"Downloading video from: {video_url}")
|
||||
video_content = _fetch_video(video_url)
|
||||
print(f"Downloaded {len(video_content)} bytes")
|
||||
video_content = _fetch_video(video_url, timeout=300)
|
||||
except Exception as e:
|
||||
print(f"Error downloading video: {str(e)}")
|
||||
return (frames_tensor, frame_count, video_fps)
|
||||
error_msg = str(e)
|
||||
print(f"Error downloading video: {error_msg}")
|
||||
|
||||
# For ComfyUI services, throw error with video URL for user recovery
|
||||
# Format error message to include the video URL for easy copy/paste
|
||||
user_error = f"Video download failed: {error_msg}\n\n📋 VIDEO URL (copy to retry manually):\n{video_url}\n\n💡 Try: Download manually or check network connection"
|
||||
|
||||
raise Exception(user_error)
|
||||
|
||||
# Extract frames
|
||||
if video_content:
|
||||
|
||||
Reference in New Issue
Block a user