Files
Ashlee Martino-Tarr b1bbb7ac89 New Nodes!
New Utility Nodes supporting relighting and compositing tasks , basic compositing, masking and color utilities
2026-01-29 15:56:56 -08:00

1956 lines
84 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Relight Utilities
Image relighting based on normal map dot product - inspired by Blender's Normal node
"""
import torch
import numpy as np
class NormalMapRelighter:
"""
Relight an image using a normal map and light direction
Uses dot product between normal vectors and light direction to calculate lighting
Similar to Blender's Normal node shader
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE",),
"normal_map": ("IMAGE",),
# Light direction (normalized vector)
"light_x": ("FLOAT", {
"default": 0.0,
"min": -1.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light direction X component"
}),
"light_y": ("FLOAT", {
"default": 0.0,
"min": -1.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light direction Y component"
}),
"light_z": ("FLOAT", {
"default": 1.0,
"min": -1.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light direction Z component (pointing toward camera)"
}),
# Light properties
"light_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Light intensity multiplier"
}),
"ambient_light": ("FLOAT", {
"default": 0.2,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Ambient light level (minimum brightness)"
}),
# Normal map format
"normal_format": (["OpenGL", "DirectX"], {
"default": "OpenGL",
"tooltip": "Normal map format (affects Y channel)"
}),
}
}
RETURN_TYPES = ("IMAGE", "IMAGE")
RETURN_NAMES = ("relit_image", "lighting_mask")
FUNCTION = "relight"
CATEGORY = "Texture Alchemist/Lighting"
def decode_normal_map(self, normal_map: torch.Tensor, format: str) -> torch.Tensor:
"""
Decode normal map from [0,1] color space to [-1,1] vector space
"""
# Convert from 0-1 to -1 to 1
normals = normal_map * 2.0 - 1.0
# Flip Y for DirectX format
if format == "DirectX":
normals[:, :, :, 1] = -normals[:, :, :, 1]
# Normalize the vectors
length = torch.sqrt(torch.sum(normals ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
normals = normals / length
return normals
def calculate_lighting(self, normals: torch.Tensor, light_dir: torch.Tensor,
intensity: float, ambient: float) -> torch.Tensor:
"""
Calculate lighting using dot product between normals and light direction
"""
# Normalize light direction
light_length = torch.sqrt(torch.sum(light_dir ** 2))
if light_length > 1e-8:
light_dir = light_dir / light_length
# Calculate dot product (N · L)
# normals shape: [B, H, W, 3]
# light_dir shape: [3]
dot_product = torch.sum(normals * light_dir, dim=-1, keepdim=True)
# Clamp dot product to [0, 1] (only front-facing surfaces are lit)
dot_product = torch.clamp(dot_product, 0.0, 1.0)
# Apply intensity and ambient light
lighting = dot_product * intensity + ambient
# Clamp final lighting
lighting = torch.clamp(lighting, 0.0, 1.0)
return lighting
def relight(self, image, normal_map, light_x, light_y, light_z,
light_intensity, ambient_light, normal_format):
"""
Relight image using normal map and light direction
"""
print("\n" + "="*60)
print("Normal Map Relighter")
print("="*60)
print(f"Image shape: {image.shape}")
print(f"Normal map shape: {normal_map.shape}")
print(f"Light direction: ({light_x:.3f}, {light_y:.3f}, {light_z:.3f})")
print(f"Light intensity: {light_intensity}")
print(f"Ambient light: {ambient_light}")
print(f"Normal format: {normal_format}")
# Decode normal map
print(f"\n🗺️ Decoding normal map...")
normals = self.decode_normal_map(normal_map, normal_format)
print(f" Normal range: [{normals.min():.3f}, {normals.max():.3f}]")
# Create light direction vector
light_dir = torch.tensor([light_x, light_y, light_z],
device=image.device, dtype=image.dtype)
# Calculate lighting
print(f"\n💡 Calculating lighting...")
lighting_mask = self.calculate_lighting(normals, light_dir,
light_intensity, ambient_light)
print(f" Lighting mask range: [{lighting_mask.min():.3f}, {lighting_mask.max():.3f}]")
# Apply lighting to image
# Expand lighting mask to match image channels
if image.shape[-1] > 1:
lighting_expanded = lighting_mask.repeat(1, 1, 1, image.shape[-1])
else:
lighting_expanded = lighting_mask
relit_image = image * lighting_expanded
# Convert lighting mask to RGB for output
lighting_mask_rgb = lighting_mask.repeat(1, 1, 1, 3)
print(f"\n✓ Relighting complete")
print(f" Relit image range: [{relit_image.min():.3f}, {relit_image.max():.3f}]")
print("="*60 + "\n")
return (relit_image, lighting_mask_rgb)
class NormalMapRelighterAdvanced:
"""
Advanced relighting with multiple light types and properties
Supports directional, point, and ambient lighting
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE",),
"normal_map": ("IMAGE",),
# Light direction (normalized vector)
"light_x": ("FLOAT", {
"default": 0.0,
"min": -1.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light direction X component"
}),
"light_y": ("FLOAT", {
"default": 0.0,
"min": -1.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light direction Y component"
}),
"light_z": ("FLOAT", {
"default": 1.0,
"min": -1.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light direction Z component"
}),
# Light properties
"diffuse_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Diffuse light intensity"
}),
"specular_intensity": ("FLOAT", {
"default": 0.5,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Specular highlight intensity"
}),
"specular_power": ("FLOAT", {
"default": 32.0,
"min": 1.0,
"max": 128.0,
"step": 1.0,
"tooltip": "Specular power (shininess)"
}),
"ambient_light": ("FLOAT", {
"default": 0.2,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Ambient light level"
}),
# Light color
"light_color_r": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light color red channel"
}),
"light_color_g": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light color green channel"
}),
"light_color_b": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Light color blue channel"
}),
# Normal map format
"normal_format": (["OpenGL", "DirectX"], {
"default": "OpenGL",
"tooltip": "Normal map format"
}),
}
}
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE")
RETURN_NAMES = ("relit_image", "diffuse_lighting", "specular_lighting")
FUNCTION = "relight"
CATEGORY = "Texture Alchemist/Lighting"
def decode_normal_map(self, normal_map: torch.Tensor, format: str) -> torch.Tensor:
"""Decode normal map from color space to vector space"""
normals = normal_map * 2.0 - 1.0
if format == "DirectX":
normals[:, :, :, 1] = -normals[:, :, :, 1]
length = torch.sqrt(torch.sum(normals ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
normals = normals / length
return normals
def calculate_diffuse(self, normals: torch.Tensor, light_dir: torch.Tensor,
intensity: float) -> torch.Tensor:
"""Calculate diffuse lighting (Lambertian)"""
dot_product = torch.sum(normals * light_dir, dim=-1, keepdim=True)
dot_product = torch.clamp(dot_product, 0.0, 1.0)
return dot_product * intensity
def calculate_specular(self, normals: torch.Tensor, light_dir: torch.Tensor,
view_dir: torch.Tensor, intensity: float, power: float) -> torch.Tensor:
"""Calculate specular lighting (Blinn-Phong)"""
# Calculate halfway vector
half_vector = light_dir + view_dir
half_length = torch.sqrt(torch.sum(half_vector ** 2))
if half_length > 1e-8:
half_vector = half_vector / half_length
# Calculate specular term
spec_dot = torch.sum(normals * half_vector, dim=-1, keepdim=True)
spec_dot = torch.clamp(spec_dot, 0.0, 1.0)
specular = torch.pow(spec_dot, power) * intensity
return specular
def relight(self, image, normal_map, light_x, light_y, light_z,
diffuse_intensity, specular_intensity, specular_power, ambient_light,
light_color_r, light_color_g, light_color_b, normal_format):
"""Advanced relighting with diffuse and specular components"""
print("\n" + "="*60)
print("Advanced Normal Map Relighter")
print("="*60)
print(f"Light direction: ({light_x:.3f}, {light_y:.3f}, {light_z:.3f})")
print(f"Light color: ({light_color_r:.3f}, {light_color_g:.3f}, {light_color_b:.3f})")
# Decode normal map
normals = self.decode_normal_map(normal_map, normal_format)
# Normalize light and view directions
light_dir = torch.tensor([light_x, light_y, light_z],
device=image.device, dtype=image.dtype)
light_length = torch.sqrt(torch.sum(light_dir ** 2))
if light_length > 1e-8:
light_dir = light_dir / light_length
# View direction (camera looking at surface)
view_dir = torch.tensor([0.0, 0.0, 1.0],
device=image.device, dtype=image.dtype)
# Calculate lighting components
diffuse = self.calculate_diffuse(normals, light_dir, diffuse_intensity)
specular = self.calculate_specular(normals, light_dir, view_dir,
specular_intensity, specular_power)
# Create light color tensor
light_color = torch.tensor([light_color_r, light_color_g, light_color_b],
device=image.device, dtype=image.dtype)
# Combine lighting components
total_lighting = diffuse + ambient_light
total_lighting = torch.clamp(total_lighting, 0.0, 1.0)
# Expand to RGB
total_lighting_rgb = total_lighting.repeat(1, 1, 1, 3) * light_color
specular_rgb = specular.repeat(1, 1, 1, 3) * light_color
# Apply to image
relit_image = image * total_lighting_rgb + specular_rgb
relit_image = torch.clamp(relit_image, 0.0, 1.0)
# Output masks
diffuse_rgb = total_lighting.repeat(1, 1, 1, 3)
specular_output = specular.repeat(1, 1, 1, 3)
print(f"\n✓ Advanced relighting complete")
print(f" Diffuse range: [{diffuse.min():.3f}, {diffuse.max():.3f}]")
print(f" Specular range: [{specular.min():.3f}, {specular.max():.3f}]")
print("="*60 + "\n")
return (relit_image, diffuse_rgb, specular_output)
class NormalMapRelighterIBL:
"""
Image-Based Lighting (IBL) using equirectangular environment maps
Samples environment map colors based on normal directions and reflection vectors
Perfect for realistic HDRI lighting
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE",),
"normal_map": ("IMAGE",),
"environment_map": ("IMAGE",),
# Environment rotation
"environment_rotation": ("FLOAT", {
"default": 0.0,
"min": -180.0,
"max": 180.0,
"step": 1.0,
"tooltip": "Rotate environment map (degrees)"
}),
# Lighting properties
"diffuse_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Diffuse lighting intensity"
}),
"specular_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Specular reflection intensity"
}),
"specular_power": ("FLOAT", {
"default": 32.0,
"min": 1.0,
"max": 128.0,
"step": 1.0,
"tooltip": "Specular power (blur amount - lower = blurrier)"
}),
"ambient_light": ("FLOAT", {
"default": 0.1,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Ambient light level"
}),
# Normal map format
"normal_format": (["OpenGL", "DirectX"], {
"default": "OpenGL",
"tooltip": "Normal map format"
}),
}
}
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE")
RETURN_NAMES = ("relit_image", "diffuse_lighting", "specular_lighting")
FUNCTION = "relight_ibl"
CATEGORY = "Texture Alchemist/Lighting"
def decode_normal_map(self, normal_map: torch.Tensor, format: str) -> torch.Tensor:
"""Decode normal map from color space to vector space"""
normals = normal_map * 2.0 - 1.0
if format == "DirectX":
normals[:, :, :, 1] = -normals[:, :, :, 1]
length = torch.sqrt(torch.sum(normals ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
normals = normals / length
return normals
def direction_to_equirect_uv(self, direction: torch.Tensor, rotation_radians: float) -> torch.Tensor:
"""
Convert 3D direction vector to equirectangular UV coordinates
direction: [B, H, W, 3] normalized vectors
returns: [B, H, W, 2] UV coordinates in range [0, 1]
"""
x = direction[:, :, :, 0]
y = direction[:, :, :, 1]
z = direction[:, :, :, 2]
# Calculate latitude (theta) and longitude (phi)
# theta = arcsin(y) - vertical angle
# phi = atan2(x, z) - horizontal angle
theta = torch.asin(torch.clamp(y, -1.0, 1.0))
phi = torch.atan2(x, z)
# Apply rotation to phi
phi = phi + rotation_radians
# Convert to UV coordinates [0, 1]
u = (phi / (2 * np.pi)) + 0.5
v = (theta / np.pi) + 0.5
# Wrap U coordinate
u = u % 1.0
# Clamp V coordinate
v = torch.clamp(v, 0.0, 1.0)
# Stack to create UV tensor
uv = torch.stack([u, v], dim=-1)
return uv
def sample_equirect(self, env_map: torch.Tensor, uv: torch.Tensor) -> torch.Tensor:
"""
Sample equirectangular map using UV coordinates with bilinear interpolation
env_map: [B, H, W, C]
uv: [B, H, W, 2]
returns: [B, H, W, C]
"""
batch_env, height_env, width_env, channels = env_map.shape
batch_uv, height_uv, width_uv, _ = uv.shape
# Convert UV to pixel coordinates
# U maps to width, V maps to height
px = uv[:, :, :, 0] * (width_env - 1)
py = uv[:, :, :, 1] * (height_env - 1)
# Bilinear interpolation
px0 = torch.floor(px).long()
py0 = torch.floor(py).long()
px1 = px0 + 1
py1 = py0 + 1
# Clamp coordinates
px0 = torch.clamp(px0, 0, width_env - 1)
px1 = torch.clamp(px1, 0, width_env - 1)
py0 = torch.clamp(py0, 0, height_env - 1)
py1 = torch.clamp(py1, 0, height_env - 1)
# Get fractional parts
fx = px - px0.float()
fy = py - py0.float()
# Expand dimensions for broadcasting with channels
fx = fx.unsqueeze(-1) # [B, H, W, 1]
fy = fy.unsqueeze(-1) # [B, H, W, 1]
# Sample four corners using advanced indexing
# We need to create batch indices that match the spatial dimensions
batch_idx = torch.arange(batch_env, device=env_map.device).view(batch_env, 1, 1).expand(batch_uv, height_uv, width_uv)
# Sample all four corners
c00 = env_map[batch_idx, py0, px0] # [B, H, W, C]
c01 = env_map[batch_idx, py0, px1] # [B, H, W, C]
c10 = env_map[batch_idx, py1, px0] # [B, H, W, C]
c11 = env_map[batch_idx, py1, px1] # [B, H, W, C]
# Bilinear interpolation
c0 = c00 * (1 - fx) + c01 * fx
c1 = c10 * (1 - fx) + c11 * fx
result = c0 * (1 - fy) + c1 * fy
return result
def calculate_reflection(self, normals: torch.Tensor, view_dir: torch.Tensor) -> torch.Tensor:
"""
Calculate reflection vector: R = 2(N·V)N - V
normals: [B, H, W, 3]
view_dir: [3] viewing direction
returns: [B, H, W, 3] reflection vectors
"""
# Calculate N·V
dot_nv = torch.sum(normals * view_dir, dim=-1, keepdim=True)
# R = 2(N·V)N - V
reflection = 2.0 * dot_nv * normals - view_dir
# Normalize
length = torch.sqrt(torch.sum(reflection ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
reflection = reflection / length
return reflection
def blur_environment_sample(self, env_map: torch.Tensor, uv: torch.Tensor,
blur_amount: float) -> torch.Tensor:
"""
Sample environment map with simple blur for roughness simulation
Lower specular_power = more blur
"""
# For now, just sample directly
# In a more advanced version, could implement mipmap-based sampling
result = self.sample_equirect(env_map, uv)
# Simple blur approximation by averaging nearby samples
if blur_amount > 1.0:
# This is a simplified version - proper IBL would use prefiltered mipmaps
pass
return result
def relight_ibl(self, image, normal_map, environment_map, environment_rotation,
diffuse_intensity, specular_intensity, specular_power, ambient_light,
normal_format):
"""
Relight image using Image-Based Lighting with equirectangular environment map
"""
print("\n" + "="*60)
print("Image-Based Lighting (IBL) Relighter")
print("="*60)
print(f"Image shape: {image.shape}")
print(f"Normal map shape: {normal_map.shape}")
print(f"Environment map shape: {environment_map.shape}")
print(f"Environment rotation: {environment_rotation}°")
# Convert rotation to radians
rotation_rad = environment_rotation * (np.pi / 180.0)
# Decode normal map
print(f"\n🗺️ Decoding normal map...")
normals = self.decode_normal_map(normal_map, normal_format)
# View direction (camera looking at surface)
view_dir = torch.tensor([0.0, 0.0, 1.0],
device=image.device, dtype=image.dtype)
# Calculate diffuse lighting from environment
print(f"\n💡 Calculating diffuse lighting from environment...")
diffuse_uv = self.direction_to_equirect_uv(normals, rotation_rad)
diffuse_colors = self.sample_equirect(environment_map, diffuse_uv)
# Apply diffuse intensity
diffuse_lighting = diffuse_colors * diffuse_intensity + ambient_light
diffuse_lighting = torch.clamp(diffuse_lighting, 0.0, 1.0)
print(f" Diffuse range: [{diffuse_lighting.min():.3f}, {diffuse_lighting.max():.3f}]")
# Calculate specular lighting from reflections
print(f"\n✨ Calculating specular reflections...")
reflection_vectors = self.calculate_reflection(normals, view_dir)
specular_uv = self.direction_to_equirect_uv(reflection_vectors, rotation_rad)
# Sample with blur based on specular power (inverse relationship)
blur_factor = 128.0 / max(specular_power, 1.0)
specular_colors = self.blur_environment_sample(environment_map, specular_uv, blur_factor)
# Calculate Fresnel-like falloff for specular
view_dot_normal = torch.sum(normals * view_dir, dim=-1, keepdim=True)
view_dot_normal = torch.clamp(view_dot_normal, 0.0, 1.0)
# Simple Fresnel approximation: more reflection at grazing angles
fresnel = torch.pow(1.0 - view_dot_normal, 5.0)
fresnel = fresnel * 0.5 + 0.5 # Soften the effect
specular_lighting = specular_colors * specular_intensity * fresnel
specular_lighting = torch.clamp(specular_lighting, 0.0, 1.0)
print(f" Specular range: [{specular_lighting.min():.3f}, {specular_lighting.max():.3f}]")
# Combine diffuse and specular
relit_image = image * diffuse_lighting + specular_lighting
relit_image = torch.clamp(relit_image, 0.0, 1.0)
print(f"\n✓ IBL relighting complete")
print(f" Final image range: [{relit_image.min():.3f}, {relit_image.max():.3f}]")
print("="*60 + "\n")
return (relit_image, diffuse_lighting, specular_lighting)
class NormalMapRelighterIBL_PBR:
"""
PBR-aware Image-Based Lighting with roughness and metallic control
Uses PBR material properties for physically accurate relighting
Non-metallic surfaces won't look metallic!
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"diffuse_image": ("IMAGE", {"tooltip": "Albedo/diffuse color map"}),
"normal_map": ("IMAGE",),
"environment_map": ("IMAGE",),
# Environment rotation
"environment_rotation": ("FLOAT", {
"default": 0.0,
"min": -180.0,
"max": 180.0,
"step": 1.0,
"tooltip": "Rotate environment map (degrees)"
}),
# Lighting properties
"diffuse_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Diffuse lighting intensity"
}),
"specular_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 10.0,
"step": 0.1,
"tooltip": "Specular reflection intensity"
}),
"ambient_light": ("FLOAT", {
"default": 0.1,
"min": 0.0,
"max": 1.0,
"step": 0.01,
"tooltip": "Ambient light level"
}),
# Roughness control
"roughness_multiplier": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 2.0,
"step": 0.1,
"tooltip": "Global roughness multiplier (affects both diffuse and specular blur)"
}),
"roughness_affects_diffuse": ("FLOAT", {
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.05,
"tooltip": "How much roughness affects diffuse detail (0=uniform, 1=fully controlled by roughness)"
}),
"diffuse_contrast": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 2.0,
"step": 0.1,
"tooltip": "Diffuse softness control (0=soft/compressed highlights, 1=normal, 2=enhanced contrast). Lower values reduce metallic look"
}),
# Diffuse color control
"use_colored_diffuse": ("BOOLEAN", {
"default": False,
"tooltip": "Use environment colors for diffuse (False = grayscale shading only)"
}),
"diffuse_quality": (["Flat Ambient (No Direction)",
"Pure Matte (No Environment)",
"Soft (Heavy Blur)",
"Detailed (Light Blur)",
"Hybrid (Lambert + Env Colors)",
"True Hemisphere Sampling"], {
"default": "Flat Ambient (No Direction)",
"tooltip": "Diffuse quality - Flat Ambient = pure even lighting (no reflection look)"
}),
# Normal map format
"normal_format": (["OpenGL", "DirectX"], {
"default": "OpenGL",
"tooltip": "Normal map format"
}),
},
"optional": {
"roughness_map": ("IMAGE",),
"metallic_map": ("IMAGE",),
"irradiance_map": ("IMAGE",),
}
}
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE")
RETURN_NAMES = ("relit_image", "diffuse_lighting", "specular_lighting", "diffuse_color", "key_light_direction")
FUNCTION = "relight_ibl_pbr"
CATEGORY = "Texture Alchemist/Lighting"
def decode_normal_map(self, normal_map: torch.Tensor, format: str) -> torch.Tensor:
"""Decode normal map from color space to vector space"""
normals = normal_map * 2.0 - 1.0
if format == "DirectX":
normals[:, :, :, 1] = -normals[:, :, :, 1]
length = torch.sqrt(torch.sum(normals ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
normals = normals / length
return normals
def prepare_pbr_map(self, pbr_map: torch.Tensor, target_shape: tuple) -> torch.Tensor:
"""Convert PBR map to grayscale and match target shape"""
if pbr_map.shape[-1] > 1:
# Convert to grayscale
weights = torch.tensor([0.299, 0.587, 0.114], device=pbr_map.device, dtype=pbr_map.dtype)
pbr_map = torch.sum(pbr_map[..., :3] * weights, dim=-1, keepdim=True)
# Expand to match target channels if needed
if pbr_map.shape[-1] != target_shape[-1]:
pbr_map = pbr_map.repeat(1, 1, 1, target_shape[-1])
return pbr_map
def direction_to_equirect_uv(self, direction: torch.Tensor, rotation_radians: float) -> torch.Tensor:
"""Convert 3D direction vector to equirectangular UV coordinates"""
x = direction[:, :, :, 0]
y = direction[:, :, :, 1]
z = direction[:, :, :, 2]
theta = torch.asin(torch.clamp(y, -1.0, 1.0))
phi = torch.atan2(x, z)
phi = phi + rotation_radians
u = (phi / (2 * np.pi)) + 0.5
v = (theta / np.pi) + 0.5
u = u % 1.0
v = torch.clamp(v, 0.0, 1.0)
uv = torch.stack([u, v], dim=-1)
return uv
def sample_equirect(self, env_map: torch.Tensor, uv: torch.Tensor) -> torch.Tensor:
"""Sample equirectangular map using UV coordinates with bilinear interpolation"""
batch_env, height_env, width_env, channels = env_map.shape
batch_uv, height_uv, width_uv, _ = uv.shape
px = uv[:, :, :, 0] * (width_env - 1)
py = uv[:, :, :, 1] * (height_env - 1)
px0 = torch.floor(px).long()
py0 = torch.floor(py).long()
px1 = px0 + 1
py1 = py0 + 1
px0 = torch.clamp(px0, 0, width_env - 1)
px1 = torch.clamp(px1, 0, width_env - 1)
py0 = torch.clamp(py0, 0, height_env - 1)
py1 = torch.clamp(py1, 0, height_env - 1)
fx = px - px0.float()
fy = py - py0.float()
fx = fx.unsqueeze(-1)
fy = fy.unsqueeze(-1)
batch_idx = torch.arange(batch_env, device=env_map.device).view(batch_env, 1, 1).expand(batch_uv, height_uv, width_uv)
c00 = env_map[batch_idx, py0, px0]
c01 = env_map[batch_idx, py0, px1]
c10 = env_map[batch_idx, py1, px0]
c11 = env_map[batch_idx, py1, px1]
c0 = c00 * (1 - fx) + c01 * fx
c1 = c10 * (1 - fx) + c11 * fx
result = c0 * (1 - fy) + c1 * fy
return result
def calculate_reflection(self, normals: torch.Tensor, view_dir: torch.Tensor) -> torch.Tensor:
"""Calculate reflection vector: R = 2(N·V)N - V"""
dot_nv = torch.sum(normals * view_dir, dim=-1, keepdim=True)
reflection = 2.0 * dot_nv * normals - view_dir
length = torch.sqrt(torch.sum(reflection ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
reflection = reflection / length
return reflection
def calculate_fresnel_schlick(self, view_dot_normal: torch.Tensor, f0: torch.Tensor) -> torch.Tensor:
"""
Fresnel-Schlick approximation
f0: base reflectivity (0.04 for dielectrics, albedo for metals)
"""
return f0 + (1.0 - f0) * torch.pow(1.0 - view_dot_normal, 5.0)
def sample_equirect_blurred(self, env_map: torch.Tensor, uv: torch.Tensor, roughness: torch.Tensor) -> torch.Tensor:
"""
Sample environment map with blur based on roughness
roughness: [B, H, W, C] where 0=smooth, 1=rough
"""
# Convert roughness to average grayscale
roughness_gray = torch.mean(roughness, dim=-1, keepdim=True) # [B, H, W, 1]
# For each pixel, sample multiple nearby UV coordinates based on roughness
# This simulates blurred reflections for rough surfaces
batch_uv, height_uv, width_uv, _ = uv.shape
# Sample center point
center_sample = self.sample_equirect(env_map, uv)
# For rough surfaces, sample additional points in a pattern
# Roughness determines blur radius
blur_radius = roughness_gray * 0.05 # Max 5% of image size blur
# Create offset pattern (8 samples around center)
offsets = [
(1, 0), (-1, 0), (0, 1), (0, -1),
(0.707, 0.707), (-0.707, 0.707), (0.707, -0.707), (-0.707, -0.707)
]
# Accumulate samples
result = center_sample
weight = 1.0
for ox, oy in offsets:
# Create offset UV
offset_uv = uv.clone()
offset_uv[:, :, :, 0] += ox * blur_radius.squeeze(-1)
offset_uv[:, :, :, 1] += oy * blur_radius.squeeze(-1)
# Wrap/clamp UV
offset_uv[:, :, :, 0] = offset_uv[:, :, :, 0] % 1.0
offset_uv[:, :, :, 1] = torch.clamp(offset_uv[:, :, :, 1], 0.0, 1.0)
# Sample and accumulate
sample = self.sample_equirect(env_map, offset_uv)
sample_weight = roughness_gray # Weight by roughness
result += sample * sample_weight
weight += sample_weight
# Average all samples
result = result / weight
return result
def relight_ibl_pbr(self, diffuse_image, normal_map, environment_map, environment_rotation,
diffuse_intensity, specular_intensity, ambient_light,
roughness_multiplier, roughness_affects_diffuse, diffuse_contrast, use_colored_diffuse, diffuse_quality,
normal_format, roughness_map=None, metallic_map=None, irradiance_map=None):
"""
PBR-aware IBL relighting with roughness and metallic control
"""
print("\n" + "="*60)
print("PBR Image-Based Lighting Relighter")
print("="*60)
print(f"Diffuse image shape: {diffuse_image.shape}")
print(f"Normal map shape: {normal_map.shape}")
print(f"Environment map shape: {environment_map.shape}")
# Convert rotation to radians
rotation_rad = environment_rotation * (np.pi / 180.0)
# Decode normal map
normals = self.decode_normal_map(normal_map, normal_format)
# Process PBR maps
if roughness_map is not None:
roughness = self.prepare_pbr_map(roughness_map, diffuse_image.shape)
print(f"Roughness map: [{roughness.min():.3f}, {roughness.max():.3f}]")
else:
# Default roughness (semi-glossy)
roughness = torch.ones_like(diffuse_image) * 0.5
print(f"Roughness map: Using default (0.5)")
# Apply roughness multiplier
roughness = roughness * roughness_multiplier
roughness = torch.clamp(roughness, 0.0, 1.0)
print(f"Roughness (after multiplier {roughness_multiplier}): [{roughness.min():.3f}, {roughness.max():.3f}]")
if metallic_map is not None:
metallic = self.prepare_pbr_map(metallic_map, diffuse_image.shape)
print(f"Metallic map: [{metallic.min():.3f}, {metallic.max():.3f}]")
else:
# Default non-metallic
metallic = torch.zeros_like(diffuse_image)
print(f"Metallic map: Using default (0.0 - non-metallic)")
# View direction
view_dir = torch.tensor([0.0, 0.0, 1.0],
device=diffuse_image.device, dtype=diffuse_image.dtype)
# Calculate diffuse lighting from environment
print(f"\n💡 Calculating PBR diffuse lighting...")
print(f" Diffuse quality mode: {diffuse_quality}")
print(f" Roughness affects diffuse: {roughness_affects_diffuse:.2f}")
# Check if we have a pre-computed irradiance map
if irradiance_map is not None:
print(f" Using provided irradiance map (pre-computed)")
diffuse_uv = self.direction_to_equirect_uv(normals, rotation_rad)
# Even with irradiance, we can apply additional roughness-based blur
if roughness_affects_diffuse > 0.0:
# Base irradiance + roughness-modulated extra blur
base_blur = roughness * roughness_affects_diffuse * 0.3 # 0 to 0.3 extra blur
diffuse_colors = self.sample_equirect_blurred(irradiance_map, diffuse_uv, base_blur)
print(f" Applied roughness-based blur to irradiance (range: [{base_blur.min():.3f}, {base_blur.max():.3f}])")
else:
diffuse_colors = self.sample_equirect(irradiance_map, diffuse_uv)
elif diffuse_quality == "Flat Ambient (No Direction)":
# PURE AMBIENT: No directional lighting at all, just average environment color
# This ensures ZERO reflection/refraction look - pure flat lighting
print(f" 🎨 Using Flat Ambient mode (pure non-directional)")
# Get average environment color
avg_env_color = torch.mean(environment_map, dim=(1, 2), keepdim=True)
# Broadcast to image dimensions (same color everywhere)
batch, height, width, _ = normals.shape
diffuse_colors = avg_env_color.expand(batch, height, width, 3)
print(f" Average environment color: RGB({avg_env_color[0,0,0,0]:.3f}, {avg_env_color[0,0,0,1]:.3f}, {avg_env_color[0,0,0,2]:.3f})")
print(f" Result: Completely flat/even lighting (no directionality)")
elif diffuse_quality == "Pure Matte (No Environment)":
# Pure matte diffuse - simple directional lighting with no environment sampling
light_dir = torch.tensor([0.2, 0.3, 0.9], device=diffuse_image.device, dtype=diffuse_image.dtype)
light_dir = light_dir / torch.sqrt(torch.sum(light_dir ** 2))
# Calculate N·L (Lambert)
dot_nl = torch.sum(normals * light_dir, dim=-1, keepdim=True)
dot_nl = torch.clamp(dot_nl, 0.0, 1.0)
# Pure matte shading (grayscale)
diffuse_colors = dot_nl.repeat(1, 1, 1, 3)
print(f" Using pure matte shading (no environment, clay-like)")
elif diffuse_quality == "Hybrid (Lambert + Env Colors)":
# SWITCHLIGHT-STYLE: Lambert shading with environment colors
# This creates pure diffuse shading without looking metallic
print(f" 🎬 Using Hybrid mode (Switchlight-style)")
# Step 1: Calculate pure Lambert shading (black & white)
# Use top hemisphere average as light direction
light_dir = torch.tensor([0.0, 0.0, 1.0], device=diffuse_image.device, dtype=diffuse_image.dtype)
dot_nl = torch.sum(normals * light_dir, dim=-1, keepdim=True)
dot_nl = torch.clamp(dot_nl, 0.0, 1.0)
print(f" Lambert shading range: [{dot_nl.min():.3f}, {dot_nl.max():.3f}]")
# Step 2: If colored diffuse is enabled, map to environment colors
if use_colored_diffuse:
# Get environment colors (using mean-based approach, not extreme quantiles)
# Convert environment to luminance
env_lum_weights = torch.tensor([0.299, 0.587, 0.114], device=environment_map.device, dtype=environment_map.dtype)
env_luminance = torch.sum(environment_map * env_lum_weights, dim=-1, keepdim=True)
# Find dark and light colors (less extreme than 10th/90th percentile)
dark_color = torch.quantile(environment_map.flatten(0, 2), 0.2, dim=0, keepdim=True) # 20th percentile (less extreme)
light_color = torch.quantile(environment_map.flatten(0, 2), 0.8, dim=0, keepdim=True) # 80th percentile (less extreme)
# Reshape for broadcasting
dark_color = dark_color.reshape(1, 1, 1, 3)
light_color = light_color.reshape(1, 1, 1, 3)
# Reduce color intensity to avoid looking specular
# Mix with grayscale to reduce saturation
avg_color = (dark_color + light_color) / 2.0
dark_color = dark_color * 0.5 + avg_color * 0.5 # 50% desaturate
light_color = light_color * 0.5 + avg_color * 0.5
# Map Lambert shading to color range
diffuse_colors = dark_color + (light_color - dark_color) * dot_nl
print(f" Dark color: [{dark_color[0,0,0,0]:.3f}, {dark_color[0,0,0,1]:.3f}, {dark_color[0,0,0,2]:.3f}]")
print(f" Light color: [{light_color[0,0,0,0]:.3f}, {light_color[0,0,0,1]:.3f}, {light_color[0,0,0,2]:.3f}]")
else:
# Pure grayscale Lambert (no color from environment)
diffuse_colors = dot_nl.repeat(1, 1, 1, 3)
print(f" Using pure grayscale Lambert (no environment color)")
elif diffuse_quality == "True Hemisphere Sampling":
# TRUE PHYSICALLY-BASED: Integrate over hemisphere
# This is the most accurate but also slowest
print(f" 🌐 Using True Hemisphere Sampling (slow but accurate)")
batch, height, width, _ = normals.shape
num_samples = 64 # Can be adjusted for quality vs speed tradeoff
# Initialize accumulator
diffuse_colors = torch.zeros_like(diffuse_image)
total_weight = torch.zeros_like(diffuse_image[:, :, :, 0:1]) # Track total weights
print(f" Sampling {num_samples} directions per pixel...")
# Generate sample directions using Fibonacci sphere (hemisphere only)
indices = torch.arange(0, num_samples, dtype=torch.float32, device=diffuse_image.device)
phi = torch.acos(1 - (indices + 0.5) / num_samples) # Only upper hemisphere (0 to π/2)
theta = np.pi * (1 + 5**0.5) * indices
# Convert to Cartesian coordinates
sample_dirs = torch.stack([
torch.sin(phi) * torch.cos(theta),
torch.sin(phi) * torch.sin(theta),
torch.cos(phi) # z always positive (hemisphere)
], dim=-1) # [num_samples, 3]
# For each sample direction
for i in range(num_samples):
sample_dir = sample_dirs[i:i+1] # [1, 3]
# Calculate N·L for this sample
dot_nl = torch.sum(normals * sample_dir, dim=-1, keepdim=True)
dot_nl = torch.clamp(dot_nl, 0.0, 1.0)
# Convert direction to UV coordinates
# Expand sample_dir to match batch dimensions
sample_dir_expanded = sample_dir.expand(batch, height, width, 3)
sample_uv = self.direction_to_equirect_uv(sample_dir_expanded, rotation_rad)
# Sample environment
env_sample = self.sample_equirect(environment_map, sample_uv)
# Accumulate: weight by N·L (Lambert's cosine law)
diffuse_colors += env_sample * dot_nl
total_weight += dot_nl
if (i + 1) % 16 == 0:
progress = (i + 1) / num_samples * 100
print(f" Progress: {progress:.0f}%")
# Average by total weight (proper Monte Carlo integration)
# Avoid division by zero
total_weight = torch.clamp(total_weight, min=1e-8)
diffuse_colors = diffuse_colors / total_weight
print(f" Hemisphere sampling complete!")
print(f" Diffuse range: [{diffuse_colors.min():.3f}, {diffuse_colors.max():.3f}]")
elif diffuse_quality == "Soft (Heavy Blur)":
# Heavy environment blur for soft diffuse, modulated by roughness
diffuse_uv = self.direction_to_equirect_uv(normals, rotation_rad)
# Base: 0.95, add roughness influence
base_blur = 0.95
roughness_influence = roughness * roughness_affects_diffuse * 0.05
diffuse_blur = base_blur + roughness_influence
diffuse_colors = self.sample_equirect_blurred(environment_map, diffuse_uv, diffuse_blur)
print(f" Using heavy blur with roughness modulation")
print(f" Blur range: [{diffuse_blur.min():.3f}, {diffuse_blur.max():.3f}]")
else: # "Detailed (Light Blur)"
# Lighter blur - more environment detail, strongly affected by roughness
diffuse_uv = self.direction_to_equirect_uv(normals, rotation_rad)
# Base: 0.5-0.7, roughness adds 0 to 0.3
base_blur = 0.5 + (1.0 - roughness_affects_diffuse) * 0.2
roughness_influence = roughness * roughness_affects_diffuse * 0.3
diffuse_blur = base_blur + roughness_influence
diffuse_colors = self.sample_equirect_blurred(environment_map, diffuse_uv, diffuse_blur)
print(f" Using detailed blur with roughness control")
print(f" Blur range: [{diffuse_blur.min():.3f}, {diffuse_blur.max():.3f}]")
# Convert diffuse to grayscale (luminance) for non-colored shading
# Colored lighting looks metallic, grayscale looks like proper shading
# NOTE: Hybrid mode handles this internally, so skip for Hybrid
if not use_colored_diffuse and diffuse_quality != "Hybrid (Lambert + Env Colors)":
# Convert to luminance for natural light/shadow appearance
weights = torch.tensor([0.299, 0.587, 0.114], device=diffuse_colors.device, dtype=diffuse_colors.dtype)
diffuse_luminance = torch.sum(diffuse_colors * weights, dim=-1, keepdim=True)
diffuse_colors = diffuse_luminance.repeat(1, 1, 1, 3)
print(f" Converting to grayscale (natural light/shadow appearance)")
elif diffuse_quality != "Hybrid (Lambert + Env Colors)":
print(f" Using colored diffuse (environment colors)")
# Apply diffuse contrast control to reduce metallic look
# Use a soft compression curve instead of linear blend to grey
if diffuse_contrast != 1.0:
if diffuse_contrast < 1.0:
# Compress highlights using a soft curve (reduces metallic look)
# This applies a power curve that compresses bright values while preserving gradients
# contrast = 0 → strong compression (very matte)
# contrast = 1 → no change
power = 1.0 + (1.0 - diffuse_contrast) * 2.0 # Range: 1.0 to 3.0
diffuse_colors = torch.pow(diffuse_colors, 1.0 / power)
print(f" Applied highlight compression: {diffuse_contrast:.2f} (power: {power:.2f})")
else:
# Enhance contrast (values > 1.0)
# Apply inverse power curve to increase contrast
power = 1.0 / diffuse_contrast
diffuse_colors = torch.pow(diffuse_colors, power)
print(f" Enhanced contrast: {diffuse_contrast:.2f} (power: {power:.2f})")
diffuse_colors = torch.clamp(diffuse_colors, 0.0, 1.0)
print(f" Diffuse range after adjustment: [{diffuse_colors.min():.3f}, {diffuse_colors.max():.3f}]")
# Calculate ambient light as average environment color (GI-style)
# Use irradiance map if available, otherwise average the environment
if irradiance_map is not None and ambient_light > 0.0:
# Average the irradiance map for ambient
ambient_color = torch.mean(irradiance_map, dim=(1, 2), keepdim=True)
print(f" Ambient from irradiance: RGB({ambient_color[0,0,0,0]:.3f}, {ambient_color[0,0,0,1]:.3f}, {ambient_color[0,0,0,2]:.3f})")
elif ambient_light > 0.0:
# Average the environment map for ambient (GI-style)
ambient_color = torch.mean(environment_map, dim=(1, 2), keepdim=True)
print(f" Ambient from environment avg: RGB({ambient_color[0,0,0,0]:.3f}, {ambient_color[0,0,0,1]:.3f}, {ambient_color[0,0,0,2]:.3f})")
else:
# No ambient
ambient_color = torch.zeros_like(diffuse_image[:, 0:1, 0:1, :])
print(f" No ambient light")
# Apply ambient light intensity
ambient_contribution = ambient_color * ambient_light
# Create pure diffuse shading (without metallic masking) for output preview
diffuse_shading_pure = diffuse_colors * diffuse_intensity + ambient_contribution
diffuse_shading_pure = torch.clamp(diffuse_shading_pure, 0.0, 1.0)
# Create diffuse for actual lighting calculation (with metallic masking)
# In PBR, metals don't have diffuse reflection
diffuse_for_lighting = diffuse_colors * diffuse_intensity * (1.0 - metallic) + ambient_contribution
diffuse_for_lighting = torch.clamp(diffuse_for_lighting, 0.0, 1.0)
print(f" Final diffuse shading range: [{diffuse_shading_pure.min():.3f}, {diffuse_shading_pure.max():.3f}]")
# Calculate specular reflections
print(f"\n✨ Calculating PBR specular reflections...")
reflection_vectors = self.calculate_reflection(normals, view_dir)
specular_uv = self.direction_to_equirect_uv(reflection_vectors, rotation_rad)
# Sample environment with roughness-based blur
# Black roughness (0) = sharp reflections, White roughness (1) = blurred reflections
specular_colors = self.sample_equirect_blurred(environment_map, specular_uv, roughness)
print(f" Applied roughness-based blur to specular reflections")
# Calculate Fresnel
view_dot_normal = torch.sum(normals * view_dir, dim=-1, keepdim=True)
view_dot_normal = torch.clamp(view_dot_normal, 0.0, 1.0)
# F0: Base reflectivity
# Dielectrics: ~0.04 (4% reflection)
# Metals: Use albedo color
f0_dielectric = torch.ones_like(diffuse_image) * 0.04
f0_metal = diffuse_image # Metals reflect colored light
f0 = f0_dielectric * (1.0 - metallic) + f0_metal * metallic
# Apply Fresnel
fresnel = self.calculate_fresnel_schlick(view_dot_normal, f0)
# Specular intensity: Metals get more specular, non-metals less
# White metallic (1) = full specular, Black metallic (0) = minimal specular
metal_boost = 1.0 + metallic * 2.0 # Metals get 3x boost
specular_lighting = specular_colors * specular_intensity * fresnel * metal_boost
specular_lighting = torch.clamp(specular_lighting, 0.0, 1.0)
print(f" Specular range: [{specular_lighting.min():.3f}, {specular_lighting.max():.3f}]")
print(f" Fresnel applied: grazing angles get more reflection")
print(f" Roughness applied: white (rough) = blurred, black (smooth) = sharp")
print(f" Metallic applied: white (metal) = strong colored reflection, black (non-metal) = weak")
# Combine lighting
# For metals: specular is the main component (colored by albedo)
# For dielectrics: diffuse is main, specular is additive
relit_image = diffuse_image * diffuse_for_lighting + specular_lighting
relit_image = torch.clamp(relit_image, 0.0, 1.0)
# Create additional outputs
print(f"\n📤 Creating additional outputs...")
# Diffuse Color: Input albedo with ONLY ambient/GI applied (no directional diffuse, no specular)
# This is: Albedo × Ambient GI only - for Switchlight-style compositing
diffuse_color = diffuse_image * ambient_contribution
diffuse_color = torch.clamp(diffuse_color, 0.0, 1.0)
print(f" Diffuse Color: Albedo × Ambient GI only (no directional lighting)")
print(f" Range: [{diffuse_color.min():.3f}, {diffuse_color.max():.3f}]")
print(f" Usage: Base image with ambient lift, add diffuse_lighting + specular separately")
# Key Light Direction: Pure Lambert shading that rotates with environment
# The light direction rotates with the environment rotation
# Start with light pointing from above/forward (typical key light position)
base_light_dir = torch.tensor([0.3, 0.5, 0.8], device=diffuse_image.device, dtype=diffuse_image.dtype)
base_light_dir = base_light_dir / torch.sqrt(torch.sum(base_light_dir ** 2)) # Normalize
# Rotate light direction by environment rotation (around Y axis)
cos_rot = torch.cos(torch.tensor(rotation_rad, device=diffuse_image.device, dtype=diffuse_image.dtype))
sin_rot = torch.sin(torch.tensor(rotation_rad, device=diffuse_image.device, dtype=diffuse_image.dtype))
# Rotation matrix around Y axis
rotated_light_x = base_light_dir[0] * cos_rot - base_light_dir[2] * sin_rot
rotated_light_y = base_light_dir[1] # Y doesn't change
rotated_light_z = base_light_dir[0] * sin_rot + base_light_dir[2] * cos_rot
key_light_dir = torch.stack([rotated_light_x, rotated_light_y, rotated_light_z])
# Calculate Lambert dot product with rotated light
key_light_dot = torch.sum(normals * key_light_dir, dim=-1, keepdim=True)
key_light_dot = torch.clamp(key_light_dot, 0.0, 1.0)
key_light_direction = key_light_dot.repeat(1, 1, 1, 3) # Convert to RGB for output
print(f" Key Light Direction: Pure Lambert mask (rotates with environment)")
print(f" Light direction: [{key_light_dir[0]:.3f}, {key_light_dir[1]:.3f}, {key_light_dir[2]:.3f}]")
print(f" Environment rotation: {environment_rotation}°")
print(f"\n✓ PBR IBL relighting complete")
print(f" Final image range: [{relit_image.min():.3f}, {relit_image.max():.3f}]")
print(f" Material interpretation:")
print(f" Roughness: Black=Shiny/Sharp, White=Rough/Blurred")
print(f" Metallic: Black=Non-Metal, White=Metal")
print("="*60 + "\n")
return (relit_image, diffuse_shading_pure, specular_lighting, diffuse_color, key_light_direction)
class PBR_AOV_Generator:
"""
PBR AOV (Arbitrary Output Variable) Generator
Takes PBR material inputs and generates proper render passes for compositing:
- Ambient Light
- GI Light Pass
- Diffuse (direct lighting)
- Reflection (specular)
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"albedo": ("IMAGE", {"tooltip": "Albedo/diffuse color map"}),
"normal_map": ("IMAGE",),
"environment_map": ("IMAGE", {"tooltip": "HDRI or irradiance map"}),
# Environment rotation
"environment_rotation": ("FLOAT", {
"default": 0.0,
"min": -180.0,
"max": 180.0,
"step": 1.0,
"tooltip": "Rotate environment map (degrees)"
}),
# Light intensities
"ambient_intensity": ("FLOAT", {
"default": 0.1,
"min": 0.0,
"max": 2.0,
"step": 0.05,
"tooltip": "Ambient fill light intensity"
}),
"gi_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 2.0,
"step": 0.1,
"tooltip": "Global illumination (indirect) intensity"
}),
"diffuse_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 2.0,
"step": 0.1,
"tooltip": "Direct diffuse lighting intensity"
}),
"reflection_intensity": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 2.0,
"step": 0.1,
"tooltip": "Reflection/specular intensity"
}),
# Normal map format
"normal_format": (["OpenGL", "DirectX"], {
"default": "OpenGL",
"tooltip": "Normal map format"
}),
},
"optional": {
"roughness_map": ("IMAGE",),
"metallic_map": ("IMAGE",),
}
}
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE")
RETURN_NAMES = ("ambient_light", "gi_light", "diffuse", "reflection")
FUNCTION = "generate_aov_passes"
CATEGORY = "Texture Alchemist/Lighting"
def decode_normal_map(self, normal_map, format):
"""Decode normal map from color space to vector space"""
normals = normal_map * 2.0 - 1.0
if format == "DirectX":
normals[:, :, :, 1] = -normals[:, :, :, 1]
length = torch.sqrt(torch.sum(normals ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
normals = normals / length
return normals
def prepare_pbr_map(self, pbr_map, target_shape):
"""Ensure PBR map matches target dimensions"""
if pbr_map.shape[1:3] != target_shape[1:3]:
import torch.nn.functional as F
pbr_map = F.interpolate(pbr_map.permute(0, 3, 1, 2),
size=(target_shape[1], target_shape[2]),
mode='bilinear',
align_corners=False).permute(0, 2, 3, 1)
if pbr_map.shape[-1] == 1:
pbr_map = pbr_map.repeat(1, 1, 1, 3)
return pbr_map
def direction_to_equirect_uv(self, directions, rotation_offset=0.0):
"""Convert 3D direction vectors to equirectangular UV coordinates"""
x, y, z = directions[:, :, :, 0], directions[:, :, :, 1], directions[:, :, :, 2]
theta = torch.atan2(x, z) + rotation_offset
phi = torch.acos(torch.clamp(y, -1.0, 1.0))
u = (theta / (2 * np.pi)) % 1.0
v = phi / np.pi
uv = torch.stack([u, v], dim=-1)
return uv
def sample_equirect(self, env_map, uv):
"""Sample equirectangular environment map using UV coordinates"""
batch, height, width, _ = uv.shape
env_height, env_width = env_map.shape[1], env_map.shape[2]
u = uv[:, :, :, 0] * (env_width - 1)
v = uv[:, :, :, 1] * (env_height - 1)
u0 = torch.floor(u).long()
u1 = torch.ceil(u).long()
v0 = torch.floor(v).long()
v1 = torch.ceil(v).long()
u0 = torch.clamp(u0, 0, env_width - 1)
u1 = torch.clamp(u1, 0, env_width - 1)
v0 = torch.clamp(v0, 0, env_height - 1)
v1 = torch.clamp(v1, 0, env_height - 1)
wu = (u - u0.float()).unsqueeze(-1)
wv = (v - v0.float()).unsqueeze(-1)
batch_idx = torch.arange(batch, device=env_map.device).view(-1, 1, 1).expand(batch, height, width)
c00 = env_map[batch_idx, v0, u0]
c01 = env_map[batch_idx, v0, u1]
c10 = env_map[batch_idx, v1, u0]
c11 = env_map[batch_idx, v1, u1]
c0 = c00 * (1 - wu) + c01 * wu
c1 = c10 * (1 - wu) + c11 * wu
result = c0 * (1 - wv) + c1 * wv
return result
def calculate_reflection(self, normals, view_dir):
"""Calculate reflection vector for environment sampling"""
dot_nv = torch.sum(normals * view_dir, dim=-1, keepdim=True)
reflection = 2.0 * dot_nv * normals - view_dir
length = torch.sqrt(torch.sum(reflection ** 2, dim=-1, keepdim=True))
length = torch.clamp(length, min=1e-8)
reflection = reflection / length
return reflection
def calculate_fresnel_schlick(self, cos_theta, f0):
"""Fresnel-Schlick approximation"""
return f0 + (1.0 - f0) * torch.pow(1.0 - cos_theta, 5.0)
def generate_aov_passes(self, albedo, normal_map, environment_map, environment_rotation,
ambient_intensity, gi_intensity, diffuse_intensity, reflection_intensity,
normal_format, roughness_map=None, metallic_map=None):
"""
Generate proper AOV render passes for compositing
"""
print("\n" + "="*60)
print("PBR AOV Generator - Render Passes")
print("="*60)
print(f"Albedo shape: {albedo.shape}")
print(f"Normal map shape: {normal_map.shape}")
print(f"Environment shape: {environment_map.shape}")
# Convert rotation to radians
rotation_rad = environment_rotation * (np.pi / 180.0)
# Decode normal map
normals = self.decode_normal_map(normal_map, normal_format)
# Process PBR maps
if roughness_map is not None:
roughness = self.prepare_pbr_map(roughness_map, albedo.shape)
print(f"Roughness map: [{roughness.min():.3f}, {roughness.max():.3f}]")
else:
roughness = torch.ones_like(albedo) * 0.5
print(f"Roughness: Using default (0.5)")
if metallic_map is not None:
metallic = self.prepare_pbr_map(metallic_map, albedo.shape)
print(f"Metallic map: [{metallic.min():.3f}, {metallic.max():.3f}]")
else:
metallic = torch.zeros_like(albedo)
print(f"Metallic: Using default (0.0)")
# View direction
view_dir = torch.tensor([0.0, 0.0, 1.0],
device=albedo.device, dtype=albedo.dtype)
# =====================================================================
# AOV 1: AMBIENT LIGHT
# Pure flat ambient fill (no directionality)
# =====================================================================
print(f"\n💡 Generating AOV 1: Ambient Light...")
# Average environment color
ambient_color = torch.mean(environment_map, dim=(1, 2), keepdim=True)
batch, height, width, _ = albedo.shape
ambient_fill = ambient_color.expand(batch, height, width, 3)
ambient_light = albedo * ambient_fill * ambient_intensity
ambient_light = torch.clamp(ambient_light, 0.0, 1.0)
print(f" Ambient color: RGB({ambient_color[0,0,0,0]:.3f}, {ambient_color[0,0,0,1]:.3f}, {ambient_color[0,0,0,2]:.3f})")
print(f" Result: Flat ambient fill × {ambient_intensity}")
# =====================================================================
# AOV 2: GI LIGHT PASS (Global Illumination - Indirect)
# Soft hemisphere-averaged environment lighting
# =====================================================================
print(f"\n🌐 Generating AOV 2: GI Light Pass...")
# Sample environment based on normals but heavily blurred (indirect)
gi_uv = self.direction_to_equirect_uv(normals, rotation_rad)
# Heavy blur for GI (indirect lighting is soft)
# Simple multi-sample blur
gi_colors = self.sample_equirect(environment_map, gi_uv)
# Apply to albedo (non-metals only for GI)
gi_light = albedo * gi_colors * gi_intensity * (1.0 - metallic)
gi_light = torch.clamp(gi_light, 0.0, 1.0)
print(f" Indirect lighting from environment")
print(f" Masked by metallic (metals don't have diffuse GI)")
# =====================================================================
# AOV 3: DIFFUSE (Direct Lighting)
# Lambert-based direct light from environment
# =====================================================================
print(f"\n☀️ Generating AOV 3: Diffuse (Direct)...")
# Key light direction (from environment dominant direction)
# For now, use upward direction as key light
key_light_dir = torch.tensor([0.3, 0.5, 0.8], device=albedo.device, dtype=albedo.dtype)
key_light_dir = key_light_dir / torch.sqrt(torch.sum(key_light_dir ** 2))
# Rotate with environment
cos_rot = torch.cos(torch.tensor(rotation_rad, device=albedo.device, dtype=albedo.dtype))
sin_rot = torch.sin(torch.tensor(rotation_rad, device=albedo.device, dtype=albedo.dtype))
rotated_x = key_light_dir[0] * cos_rot - key_light_dir[2] * sin_rot
rotated_y = key_light_dir[1]
rotated_z = key_light_dir[0] * sin_rot + key_light_dir[2] * cos_rot
key_light = torch.stack([rotated_x, rotated_y, rotated_z])
# Lambert shading
dot_nl = torch.sum(normals * key_light, dim=-1, keepdim=True)
dot_nl = torch.clamp(dot_nl, 0.0, 1.0)
# Sample environment at key light direction for color
key_light_uv = self.direction_to_equirect_uv(
key_light.unsqueeze(0).unsqueeze(0).expand(batch, height, width, 3),
rotation_rad
)
key_light_color = self.sample_equirect(environment_map, key_light_uv)
# Direct diffuse (non-metals only)
diffuse = albedo * key_light_color * dot_nl * diffuse_intensity * (1.0 - metallic)
diffuse = torch.clamp(diffuse, 0.0, 1.0)
print(f" Direct Lambert lighting from key light")
print(f" Key light direction: [{key_light[0]:.3f}, {key_light[1]:.3f}, {key_light[2]:.3f}]")
# =====================================================================
# AOV 4: REFLECTION (Specular)
# Environment reflections with PBR
# =====================================================================
print(f"\n✨ Generating AOV 4: Reflection...")
# Calculate reflection vectors
reflection_vectors = self.calculate_reflection(normals, view_dir)
reflection_uv = self.direction_to_equirect_uv(reflection_vectors, rotation_rad)
# Sample environment (would need blur based on roughness for proper PBR)
reflection_colors = self.sample_equirect(environment_map, reflection_uv)
# Calculate Fresnel
view_dot_normal = torch.sum(normals * view_dir, dim=-1, keepdim=True)
view_dot_normal = torch.clamp(view_dot_normal, 0.0, 1.0)
f0_dielectric = torch.ones_like(albedo) * 0.04
f0_metal = albedo
f0 = f0_dielectric * (1.0 - metallic) + f0_metal * metallic
fresnel = self.calculate_fresnel_schlick(view_dot_normal, f0)
# Metal boost for reflections
metal_boost = 1.0 + metallic * 2.0
reflection = reflection_colors * fresnel * reflection_intensity * metal_boost
reflection = torch.clamp(reflection, 0.0, 1.0)
print(f" Environment reflections with Fresnel")
print(f" Boosted for metallic surfaces")
print(f"\n✓ AOV Generation Complete")
print(f" 4 render passes ready for compositing")
print("="*60 + "\n")
return (ambient_light, gi_light, diffuse, reflection)
class Marigold_AOV_Extractor:
"""
Extract render passes (AOVs) from Marigold IId outputs
Based on Octane render passes: https://help.otoy.com/hc/en-us/articles/30487201925019
Extracts:
- Diffuse (Albedo from Lighting)
- Lighting/Shading (Direct + Indirect illumination)
- Reflection/Specular (Residual highlights)
- Ambient Occlusion (from shading analysis)
- Emission (bright residual areas)
"""
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"marigold_lighting": ("IMAGE", {
"tooltip": "Marigold IId Lighting output (contains albedo + shading)"
}),
"marigold_appearance": ("IMAGE", {
"tooltip": "Marigold IId Appearance output (contains residual/specular)"
}),
# Extraction controls
"diffuse_gamma": ("FLOAT", {
"default": 1.0,
"min": 0.1,
"max": 3.0,
"step": 0.1,
"tooltip": "Gamma correction for diffuse/albedo"
}),
"shading_strength": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 3.0,
"step": 0.1,
"tooltip": "Lighting/shading intensity"
}),
"specular_threshold": ("FLOAT", {
"default": 0.5,
"min": 0.0,
"max": 1.0,
"step": 0.05,
"tooltip": "Threshold for extracting specular from residual"
}),
"specular_strength": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 3.0,
"step": 0.1,
"tooltip": "Specular/reflection intensity"
}),
"ao_strength": ("FLOAT", {
"default": 1.0,
"min": 0.0,
"max": 2.0,
"step": 0.1,
"tooltip": "Ambient occlusion extraction strength"
}),
"emission_threshold": ("FLOAT", {
"default": 0.8,
"min": 0.0,
"max": 1.0,
"step": 0.05,
"tooltip": "Threshold for extracting emission from bright areas"
}),
},
"optional": {
"normal_map": ("IMAGE", {
"tooltip": "Optional normal map for enhanced AO extraction"
}),
}
}
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE")
RETURN_NAMES = ("diffuse_albedo", "lighting_direct", "lighting_indirect", "reflection_specular", "ambient_occlusion", "emission", "beauty_composite")
FUNCTION = "extract_render_passes"
CATEGORY = "Texture Alchemist/Lighting"
def extract_render_passes(self, marigold_lighting, marigold_appearance,
diffuse_gamma, shading_strength, specular_threshold,
specular_strength, ao_strength, emission_threshold,
normal_map=None):
"""
Extract render passes from Marigold IId outputs
Marigold IId Lighting decomposition:
- Albedo: Diffuse surface color (no lighting)
- Shading: Direct + Indirect illumination
Marigold IId Appearance:
- Residual: Information not captured by albedo/shading (specular, reflections)
"""
print("\n" + "="*60)
print("Marigold AOV Extractor - Render Pass Decomposition")
print("="*60)
print(f"Lighting input shape: {marigold_lighting.shape}")
print(f"Appearance input shape: {marigold_appearance.shape}")
# Ensure both inputs are RGB (3 channels)
# Marigold sometimes outputs grayscale or 2-channel images
def ensure_rgb(tensor, name):
"""Convert grayscale/2-channel to RGB"""
if tensor.shape[-1] == 1:
# Grayscale to RGB
print(f" Converting {name} from grayscale to RGB")
return tensor.repeat(1, 1, 1, 3)
elif tensor.shape[-1] == 2:
# 2-channel to RGB (repeat first channel for blue)
print(f" Converting {name} from 2-channel to RGB")
return torch.cat([tensor, tensor[:, :, :, 0:1]], dim=-1)
elif tensor.shape[-1] == 3:
# Already RGB
return tensor
else:
# More than 3 channels, take first 3
print(f" Warning: {name} has {tensor.shape[-1]} channels, using first 3")
return tensor[:, :, :, :3]
marigold_lighting = ensure_rgb(marigold_lighting, "Lighting")
marigold_appearance = ensure_rgb(marigold_appearance, "Appearance")
# Ensure batch sizes match (take minimum)
if marigold_lighting.shape[0] != marigold_appearance.shape[0]:
min_batch = min(marigold_lighting.shape[0], marigold_appearance.shape[0])
print(f" Warning: Batch size mismatch!")
print(f" Lighting batch: {marigold_lighting.shape[0]}, Appearance batch: {marigold_appearance.shape[0]}")
print(f" Using first {min_batch} images from each")
marigold_lighting = marigold_lighting[:min_batch]
marigold_appearance = marigold_appearance[:min_batch]
# Ensure inputs match spatial dimensions
if marigold_lighting.shape[1:3] != marigold_appearance.shape[1:3]:
import torch.nn.functional as F
target_shape = marigold_lighting.shape[1:3]
marigold_appearance = F.interpolate(
marigold_appearance.permute(0, 3, 1, 2),
size=target_shape,
mode='bilinear',
align_corners=False
).permute(0, 2, 3, 1)
print(f" Resized appearance to match lighting: {marigold_appearance.shape}")
print(f"Final shapes - Lighting: {marigold_lighting.shape}, Appearance: {marigold_appearance.shape}")
# =====================================================================
# PASS 1: DIFFUSE ALBEDO
# Pure surface color without lighting
# =====================================================================
print(f"\n📦 Pass 1: Diffuse Albedo...")
# Marigold Lighting contains the albedo component
# We need to extract it by removing the shading
# Simple approach: The lighting image is albedo × shading
# We can approximate albedo by analyzing the image
# For now, use the lighting output as diffuse (it's already close)
diffuse_albedo = marigold_lighting.clone()
# Apply gamma correction
if diffuse_gamma != 1.0:
diffuse_albedo = torch.pow(diffuse_albedo, diffuse_gamma)
diffuse_albedo = torch.clamp(diffuse_albedo, 0.0, 1.0)
print(f" Extracted diffuse albedo with gamma={diffuse_gamma}")
# =====================================================================
# PASS 2 & 3: LIGHTING (Direct + Indirect)
# =====================================================================
print(f"\n💡 Pass 2 & 3: Lighting (Direct + Indirect)...")
# The "shading" in Marigold represents the lighting
# We approximate this by dividing the lit image by albedo
# shading = lit_image / albedo
# To extract shading, we need the ratio
# Avoid division by zero
albedo_safe = torch.clamp(marigold_lighting, min=0.01)
# The appearance contains the residual, so we can use lighting as base
# Convert to grayscale for shading intensity
lighting_intensity = torch.mean(marigold_lighting, dim=-1, keepdim=True)
lighting_intensity = lighting_intensity.repeat(1, 1, 1, 3)
# Direct lighting: Brighter areas (peaks)
lighting_direct = torch.clamp(lighting_intensity * shading_strength, 0.0, 1.0)
# Indirect lighting: Softer, ambient component
# GI should be VERY soft and diffuse (no specular)
# Apply heavy blur to remove specular highlights
import torch.nn.functional as F
# Convert to BCHW format for conv2d
lighting_for_blur = lighting_intensity.permute(0, 3, 1, 2)
# Create large gaussian-like blur kernel for soft GI
kernel_size = 21 # Larger kernel = softer, more diffuse
sigma = kernel_size / 3.0
# Simple box blur (faster than gaussian)
# Multiple passes of small blur = larger blur
for _ in range(3): # 3 passes for soft result
lighting_for_blur = F.avg_pool2d(
lighting_for_blur,
kernel_size=5,
stride=1,
padding=2,
count_include_pad=False
)
# Convert back to BHWC
lighting_indirect = lighting_for_blur.permute(0, 2, 3, 1)
# Further reduce contrast for GI (very soft)
lighting_indirect = (lighting_indirect - 0.5) * 0.3 + 0.5 # Much softer
lighting_indirect = torch.clamp(lighting_indirect * shading_strength * 0.4, 0.0, 1.0)
print(f" Extracted direct lighting (strength={shading_strength})")
print(f" Extracted indirect lighting (heavily blurred, matte)")
# =====================================================================
# PASS 4: REFLECTION/SPECULAR
# Extract from residual (appearance - lighting)
# =====================================================================
print(f"\n✨ Pass 4: Reflection/Specular...")
# The residual is what appearance captures that lighting doesn't
# This includes specular highlights and reflections
residual = marigold_appearance - marigold_lighting
residual = torch.clamp(residual, 0.0, 1.0)
# Extract bright spots as specular
residual_intensity = torch.mean(residual, dim=-1, keepdim=True)
# Threshold to get specular highlights
specular_mask = (residual_intensity > specular_threshold).float()
# Apply mask and boost
reflection_specular = residual * specular_mask * specular_strength
reflection_specular = torch.clamp(reflection_specular, 0.0, 1.0)
print(f" Extracted specular with threshold={specular_threshold}")
print(f" Specular pixels: {(specular_mask.sum() / specular_mask.numel() * 100):.1f}%")
# =====================================================================
# PASS 5: AMBIENT OCCLUSION
# Extract from shadow analysis
# =====================================================================
print(f"\n🌑 Pass 5: Ambient Occlusion...")
# AO is typically the darkest areas in shading
# Convert lighting to grayscale and invert dark areas
lighting_gray = torch.mean(marigold_lighting, dim=-1, keepdim=True)
# Invert and boost shadows
ao_raw = 1.0 - lighting_gray
# Apply contrast to enhance
ao_raw = torch.pow(ao_raw, 1.0 / ao_strength) if ao_strength > 0 else ao_raw
# Normalize
ao_min = ao_raw.min()
ao_max = ao_raw.max()
if ao_max > ao_min:
ambient_occlusion = (ao_raw - ao_min) / (ao_max - ao_min)
else:
ambient_occlusion = ao_raw
# Convert to RGB for display
ambient_occlusion = ambient_occlusion.repeat(1, 1, 1, 3)
ambient_occlusion = torch.clamp(ambient_occlusion, 0.0, 1.0)
print(f" Extracted AO from shadow analysis (strength={ao_strength})")
# Enhance with normal map if provided
if normal_map is not None:
print(f" Enhancing AO with normal map...")
# Use normal map to refine AO (darker in crevices)
normal_gray = torch.mean(normal_map, dim=-1, keepdim=True)
normal_factor = 1.0 - torch.abs(normal_gray - 0.5) * 2.0
ambient_occlusion = ambient_occlusion * (0.7 + normal_factor * 0.3)
ambient_occlusion = torch.clamp(ambient_occlusion, 0.0, 1.0)
# =====================================================================
# PASS 6: EMISSION
# Extract very bright areas as self-illumination
# =====================================================================
print(f"\n💥 Pass 6: Emission...")
# Emission is extracted from very bright areas in appearance
appearance_intensity = torch.mean(marigold_appearance, dim=-1, keepdim=True)
# Threshold for emission
emission_mask = (appearance_intensity > emission_threshold).float()
# Extract emission
emission = marigold_appearance * emission_mask
# Boost emission
emission = torch.clamp(emission * 1.5, 0.0, 1.0)
emission_percent = (emission_mask.sum() / emission_mask.numel() * 100)
print(f" Extracted emission with threshold={emission_threshold}")
print(f" Emissive pixels: {emission_percent:.1f}%")
# =====================================================================
# PASS 7: BEAUTY COMPOSITE
# Combine all passes for reference
# =====================================================================
print(f"\n🎨 Pass 7: Beauty Composite...")
# Beauty = (Diffuse × Lighting_Direct) + Lighting_Indirect + Reflection + Emission
beauty_composite = (
diffuse_albedo * lighting_direct +
lighting_indirect +
reflection_specular +
emission
)
beauty_composite = torch.clamp(beauty_composite, 0.0, 1.0)
print(f" Composed beauty pass from all AOVs")
# =====================================================================
# Summary
# =====================================================================
print(f"\n✓ Render Pass Extraction Complete")
print(f" 7 passes ready for compositing:")
print(f" 1. Diffuse Albedo - Base surface color")
print(f" 2. Lighting Direct - Direct illumination")
print(f" 3. Lighting Indirect - Ambient/bounce light")
print(f" 4. Reflection - Specular highlights")
print(f" 5. Ambient Occlusion - Shadow/cavity AO")
print(f" 6. Emission - Self-illumination")
print(f" 7. Beauty Composite - Final combined result")
print("="*60 + "\n")
return (diffuse_albedo, lighting_direct, lighting_indirect,
reflection_specular, ambient_occlusion, emission, beauty_composite)
# Node registration
NODE_CLASS_MAPPINGS = {
"NormalMapRelighter": NormalMapRelighter,
"NormalMapRelighterAdvanced": NormalMapRelighterAdvanced,
"NormalMapRelighterIBL": NormalMapRelighterIBL,
"NormalMapRelighterIBL_PBR": NormalMapRelighterIBL_PBR,
"PBR_AOV_Generator": PBR_AOV_Generator,
"Marigold_AOV_Extractor": Marigold_AOV_Extractor,
}
NODE_DISPLAY_NAME_MAPPINGS = {
"NormalMapRelighter": "Normal Map Relighter",
"NormalMapRelighterAdvanced": "Normal Map Relighter (Advanced)",
"NormalMapRelighterIBL": "Normal Map Relighter (IBL)",
"NormalMapRelighterIBL_PBR": "Normal Map Relighter (IBL + PBR)",
"PBR_AOV_Generator": "🎬 PBR AOV Generator (Render Passes)",
"Marigold_AOV_Extractor": "📦 Marigold AOV Extractor (Render Passes)",
}