feat: add LoggingUtils class with base64 string truncation

- Prevent console spam from base64 image data in API payloads
- Recursively truncate base64 strings in dictionaries and lists
- Show first 50 chars + truncation message for debugging clarity
This commit is contained in:
pixelworld AI
2025-09-14 13:15:08 -05:00
parent 0c5428a57e
commit 191264270c
3 changed files with 143 additions and 15 deletions
+3 -3
View File
@@ -3,7 +3,7 @@ import requests
import torch
import numpy as np
from PIL import Image
from .byteplus_utils import BytePlusConfig, BytePlusImageUtils
from .byteplus_utils import BytePlusConfig, BytePlusImageUtils, LoggingUtils
class SeedResultProcessor:
@@ -67,7 +67,7 @@ class SeedImageApiHandler:
print(f"Making Seed image request to: {config.get_base_url()}/images/generations")
print(f"Model: {model}")
print(f"Requested batch size (n): {payload.get('n', 1)}")
print(f"Payload: {payload}")
LoggingUtils.safe_log_payload(payload, "Payload")
try:
response = requests.post(
@@ -79,7 +79,7 @@ class SeedImageApiHandler:
if response.status_code == 200:
result = response.json()
print(f"Seed image generation successful: {result}")
LoggingUtils.safe_log_payload(result, "Seed image generation successful")
return result
else:
print(f"Seed image generation error: {response.status_code} - {response.text}")
+38 -3
View File
@@ -6,6 +6,7 @@ import os
import time
import tempfile
from typing import Optional, Dict, Any
import copy
import numpy as np
import requests
@@ -13,6 +14,40 @@ import torch
from PIL import Image
class LoggingUtils:
"""Utility functions for safe logging that truncates base64 strings."""
@staticmethod
def truncate_base64_in_dict(data, max_length=50):
"""Recursively truncate base64 strings in a dictionary for logging."""
if isinstance(data, dict):
truncated = {}
for key, value in data.items():
truncated[key] = LoggingUtils.truncate_base64_in_dict(value, max_length)
return truncated
elif isinstance(data, list):
return [LoggingUtils.truncate_base64_in_dict(item, max_length) for item in data]
elif isinstance(data, str):
# Check if string looks like base64 data URI
if data.startswith('data:image/') and 'base64,' in data and len(data) > 100:
# Find the base64 part
base64_start = data.find('base64,') + 7
prefix = data[:base64_start]
base64_part = data[base64_start:]
if len(base64_part) > max_length:
truncated_b64 = base64_part[:max_length] + f"...[{len(base64_part)-max_length} chars truncated]"
return prefix + truncated_b64
return data
else:
return data
@staticmethod
def safe_log_payload(payload, description="Payload"):
"""Safely log a payload by truncating base64 strings."""
safe_payload = LoggingUtils.truncate_base64_in_dict(copy.deepcopy(payload))
print(f"{description}: {safe_payload}")
class BytePlusConfig:
"""Singleton class to handle BytePlus configuration and client setup."""
@@ -141,7 +176,7 @@ class BytePlusApiHandler:
print(f"API Key length: {len(config.get_api_key()) if config.get_api_key() else 0}")
print(f"API Key starts with: {config.get_api_key()[:10] if config.get_api_key() else 'None'}...")
print(f"Headers: {headers}")
print(f"Payload: {payload}")
LoggingUtils.safe_log_payload(payload, "Payload")
if callback_url:
payload["callback_url"] = callback_url
@@ -208,8 +243,8 @@ class BytePlusApiHandler:
print(f"Task {task_id} status: {status}")
if status == "succeeded":
# Debug: Print the full response to understand the structure
print(f"Full successful response: {result}")
# Debug: Print the full response to understand the structure (with base64 truncation)
LoggingUtils.safe_log_payload(result, "Full successful response")
# Look for video URL in the result - try multiple possible paths
if "content" in result and "video_url" in result["content"]:
+102 -9
View File
@@ -3,6 +3,7 @@ import torch
import requests
import tempfile
import numpy as np
import time
from PIL import Image
# Try to import opencv - if not available, provide error message
@@ -15,9 +16,97 @@ except ImportError:
print("To install: pip install opencv-python>=4.5.0")
def _fetch_video(url, stream=True):
"""Download video content from URL."""
return requests.get(url, stream=stream).content
def _fetch_video(url, timeout=300):
"""Download video content from URL with streaming and proper total timeout enforcement."""
try:
print(f"Starting video download from: {url}")
print(f"Total timeout: {timeout} seconds")
# Record start time for total timeout enforcement
start_time = time.time()
# Set headers to mimic a browser request
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Accept': 'video/*, */*',
'Accept-Encoding': 'identity', # Disable compression to avoid issues
'Connection': 'keep-alive'
}
# Start streaming request with connection timeout only
# We'll handle total timeout manually
try:
response = requests.get(url, stream=True, timeout=30, headers=headers) # 30s connection timeout only
response.raise_for_status() # Raise exception for bad status codes
except requests.exceptions.Timeout:
raise Exception("Connection timeout - could not connect to server within 30 seconds")
# Check if we've already exceeded timeout during connection
elapsed = time.time() - start_time
if elapsed >= timeout:
response.close()
raise Exception(f"Total timeout exceeded during connection ({elapsed:.1f}s >= {timeout}s)")
# Get file size if available
total_size = response.headers.get('content-length')
if total_size:
total_size = int(total_size)
print(f"Video size: {total_size / (1024*1024):.2f} MB")
else:
print("Video size: Unknown")
print(f"Starting download, will timeout after {timeout} seconds...")
# Download in chunks with strict total timeout enforcement
video_content = bytearray()
downloaded = 0
chunk_size = 1024 * 1024 # 1MB chunks
last_progress_logged = 0
last_timeout_check = start_time
try:
for chunk in response.iter_content(chunk_size=chunk_size, decode_unicode=False):
# Check total timeout every few chunks (not every single chunk for performance)
current_time = time.time()
if current_time - last_timeout_check > 5: # Check every 5 seconds
elapsed = current_time - start_time
if elapsed >= timeout:
response.close()
raise Exception(f"Download timeout exceeded: {elapsed:.1f}s >= {timeout}s (downloaded {downloaded/(1024*1024):.2f} MB)")
last_timeout_check = current_time
if chunk:
video_content.extend(chunk)
downloaded += len(chunk)
# Log progress every 10% or every 10MB, whichever comes first
if total_size:
progress = (downloaded / total_size) * 100
if progress - last_progress_logged >= 10:
elapsed = time.time() - start_time
print(f"Download progress: {progress:.1f}% ({downloaded / (1024*1024):.2f} MB) - {elapsed:.1f}s elapsed")
last_progress_logged = progress
else:
# Log every 10MB when size is unknown
mb_downloaded = downloaded / (1024*1024)
if mb_downloaded - last_progress_logged >= 10:
elapsed = time.time() - start_time
print(f"Downloaded: {mb_downloaded:.2f} MB - {elapsed:.1f}s elapsed")
last_progress_logged = mb_downloaded
finally:
response.close()
final_elapsed = time.time() - start_time
print(f"Download complete: {len(video_content)} bytes in {final_elapsed:.1f} seconds")
return bytes(video_content)
except requests.exceptions.Timeout:
raise Exception(f"Download timeout after {timeout}s - video may be too large or connection too slow")
except requests.exceptions.RequestException as e:
raise Exception(f"Download failed: {str(e)}")
except Exception as e:
raise Exception(f"Unexpected error during download: {str(e)}")
class VideoToFrames:
@@ -62,14 +151,18 @@ class VideoToFrames:
frame_count = 0
video_fps = 0.0
# Download video content
# Download video content (5 minute timeout)
try:
print(f"Downloading video from: {video_url}")
video_content = _fetch_video(video_url)
print(f"Downloaded {len(video_content)} bytes")
video_content = _fetch_video(video_url, timeout=300)
except Exception as e:
print(f"Error downloading video: {str(e)}")
return (frames_tensor, frame_count, video_fps)
error_msg = str(e)
print(f"Error downloading video: {error_msg}")
# For ComfyUI services, throw error with video URL for user recovery
# Format error message to include the video URL for easy copy/paste
user_error = f"Video download failed: {error_msg}\n\n📋 VIDEO URL (copy to retry manually):\n{video_url}\n\n💡 Try: Download manually or check network connection"
raise Exception(user_error)
# Extract frames
if video_content: