nodes/willchil-ComfyUI-Environment-Visualizer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added drag rotate with VR controllers

Browse Source
This commit is contained in:
William Chilcote
2024-10-06 18:12:28 -07:00
parent 5e92b23ac9
commit 8027681f11
2 changed files with 90 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -5,8 +5,8 @@ This ComfyUI node pack allows the user to take a panoramic image and a correspon
When this node pack is installed, ComfyUI will launch a web server on the same machine when ComfyUI is initialized. By default, the server will be hosted on port 4443, but this can be changed by modifying the value of `SERVER_PORT` at the top of `server.py`. As WebXR requires an HTTPS environment to run, a self-signed certificate is generated on first launch, which will display a warning when loaded on most browsers. The web page includes a dropdown to pick from previously saved environments, as well as several controls to adjust the presentation of the environment. The user can turn left or right in the environment with the joysticks on a VR controller, or with A/D and Left/Right on a keyboard. As ComfyUI will always launch this web server on start whenever the node pack is active, you may consider disabling the pack when it is not in use.
### Included ComfyUI nodes
- **Environment Visualizer**: This output node accepts two image inputs, a 2:1 image for the panoramic skybox, and a 2:1 image for the depth map. The node includes a text input for the environment name, and a toggle to automatically launch the generated visualization in a new tab when the workflow has completed. Running this node will automatically save the texture and depth map images into a new subdirectory in the `/environments` directory with the provided name. The `depth` input is optional, and the environment will be displayed as a flat skybox if it is omitted.
- **Interpolate Edges**: This node takes an image, and linearly interpolates the edge pixel values along the vertical edges such that they continuously blend to the opposite edge. This is intended for smoothly blending the seams of the generated depth maps to remove discontinuities. The smoothing will be applied over the given number of pixels on each side. While this approach will work on the panoramic texture or any other image, it is more noticeable in color, and is primarily intended for use with the depth map.
- **EnvironmentVisualizer**: This output node accepts two image inputs, a 2:1 image for the panoramic skybox, and a 2:1 image for the depth map. The node includes a text input for the environment name, and a toggle to automatically launch the generated visualization in a new tab when the workflow has completed. Running this node will automatically save the texture and depth map images into a new subdirectory in the `/environments` directory with the provided name. The `depth` input is optional, and the environment will be displayed as a flat skybox if it is omitted.
- **InterpolateEdges**: This node takes an image, and linearly interpolates the edge pixel values along the vertical edges such that they continuously blend to the opposite edge. This is intended for smoothly blending the seams of the generated depth maps to remove discontinuities. The smoothing will be applied over the given number of pixels on each side. While this approach will work on the panoramic texture or any other image, it is more noticeable in color, and is primarily intended for use with the depth map.
- **MapEquirectangular**: This node accepts an image, alongside camera parameters such as its FOV and orientation, and projects the image onto an equirectangular image of a given width. Pixels in the resulting equirectangular image that are not mapped from the input image will be black and have zero opacity. This allows the image to be split into both the RGB texture and mask to simplify inpainting. Note that processing the projection with a yaw of 180 degrees will split the image across the horizontal edges, which may help with creating seamless panoramas.
## Generation tips
script.js
+88 -15
View File
@@ -2,7 +2,7 @@ import * as THREE from 'https://cdn.jsdelivr.net/npm/three@0.168.0/build/three.m
import { VRButton } from 'https://cdn.jsdelivr.net/npm/three@0.168.0/examples/jsm/webxr/VRButton.js';
let camera, scene, renderer;
let environmentTexture, depthTexture = null; // Initialize depthTexture as null
let environmentTexture, depthTexture = null;
let minDistance = 2.0;
let depthRange = 3.0;
let resolution = 2;
@@ -10,12 +10,9 @@ let meshDirty = false;
let environmentMesh;
let rotationGroup;
const rotation_angle = 30; // in degrees
const rotation_angle = 30; // Degrees
const resolutions = [512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192];
// Initialize a Map to track controller states for snap rotation
const controllerStates = new Map();
// Variables for mouse interaction
let isUserInteracting = false,
onPointerDownMouseX = 0,
@@ -29,6 +26,11 @@ let isUserInteracting = false,
let rotationSensitivityX = 0.1; // Horizontal rotation sensitivity
let rotationSensitivityY = 0.1; // Vertical rotation sensitivity
const controllers = []; // Array to hold controllers
const controllerStates = new Map(); // Map to hold controller states
const max_lat = 85;
init();
animate();
@@ -70,7 +72,7 @@ async function init() {
setMeshDirty(true);
});
const directories = await fetchEnvironments(); // Replace with actual directory fetch
const directories = await fetchEnvironments();
directories.forEach(dir => {
const option = document.createElement('option');
option.value = dir;
@@ -106,10 +108,19 @@ async function init() {
renderer.domElement.addEventListener('mouseup', onPointerUp, false);
renderer.domElement.addEventListener('mouseleave', onPointerUp, false);
// Optional: Event listeners for touch interaction (mobile devices)
// Event listeners for touch interaction (mobile devices)
renderer.domElement.addEventListener('touchstart', onPointerDown, false);
renderer.domElement.addEventListener('touchmove', onPointerMove, false);
renderer.domElement.addEventListener('touchend', onPointerUp, false);
// Initialize controllers
for (let i = 0; i <= 1; i++) {
const controller = renderer.xr.getController(i);
controller.addEventListener('selectstart', onSelectStart);
controller.addEventListener('selectend', onSelectEnd);
scene.add(controller);
controllers.push(controller);
}
}
async function fetchEnvironments() {
@@ -321,6 +332,8 @@ function processControllerInput() {
const session = renderer.xr.getSession();
if (session) {
const inputSources = session.inputSources;
// Handle joystick snap rotation
inputSources.forEach(inputSource => {
if (inputSource && inputSource.gamepad) {
// Initialize state for this controller if not already done
@@ -352,6 +365,52 @@ function processControllerInput() {
}
}
});
// Handle trigger drag rotation
controllers.forEach(controller => {
if (controller.userData.isSelecting) {
if (controller.userData.needsInitialPosition) {
// Record initial positions
const initialHeadPosition = new THREE.Vector3().setFromMatrixPosition(camera.matrixWorld);
const initialControllerPosition = new THREE.Vector3().setFromMatrixPosition(controller.matrixWorld);
// Compute initial vector from head to controller
const initialControllerVector = new THREE.Vector3().subVectors(initialControllerPosition, initialHeadPosition);
// Project onto XZ plane
initialControllerVector.y = 0;
if (initialControllerVector.lengthSq() === 0) return; // Avoid division by zero
initialControllerVector.normalize();
controller.userData.initialControllerVectorXZ = initialControllerVector;
controller.userData.needsInitialPosition = false;
controller.userData.initialLon = lon; // Store initial lon
} else {
// Get current positions
const currentHeadPosition = new THREE.Vector3().setFromMatrixPosition(camera.matrixWorld);
const currentControllerPosition = new THREE.Vector3().setFromMatrixPosition(controller.matrixWorld);
// Compute current vector from head to controller
const currentControllerVector = new THREE.Vector3().subVectors(currentControllerPosition, currentHeadPosition);
// Project onto XZ plane
currentControllerVector.y = 0;
if (currentControllerVector.lengthSq() === 0) return; // Avoid division by zero
currentControllerVector.normalize();
// Compute angle difference between initial and current vectors
const initialVector = controller.userData.initialControllerVectorXZ;
const dot = initialVector.dot(currentControllerVector);
const crossY = initialVector.x * currentControllerVector.z - initialVector.z * currentControllerVector.x; // Cross product in Y
let angle = Math.atan2(crossY, dot); // Angle in radians
angle = THREE.MathUtils.radToDeg(angle); // Convert to degrees
// Invert the angle to change rotation direction
lon = controller.userData.initialLon - angle;
}
}
});
}
}
@@ -383,25 +442,26 @@ function onKeyDown(event) {
// Rotate the scene left
function rotateSceneLeft() {
lon -= rotation_angle; // Adjust lon instead of rotationGroup.rotation.y
lon -= rotation_angle;
}
// Rotate the scene right
function rotateSceneRight() {
lon += rotation_angle; // Adjust lon instead of rotationGroup.rotation.y
lon += rotation_angle;
}
// Optional: Rotate the scene up and down using keyboard
// Rotate the scene up
function rotateSceneUp() {
lat += rotation_angle; // Adjust lat
lat += rotation_angle;
// Clamp latitude to prevent flipping over the poles
lat = Math.max(-85, Math.min(85, lat));
lat = Math.max(-max_lat, Math.min(max_lat, lat));
}
// Rotate the scene down
function rotateSceneDown() {
lat -= rotation_angle; // Adjust lat
lat -= rotation_angle;
// Clamp latitude to prevent flipping over the poles
lat = Math.max(-85, Math.min(85, lat));
lat = Math.max(-max_lat, Math.min(max_lat, lat));
}
// Mouse and touch event handlers for rotation
@@ -436,10 +496,23 @@ function onPointerMove(event) {
lat = (onPointerDownMouseY - clientY) * rotationSensitivityY + onPointerDownLat;
// Clamp latitude to prevent flipping over the poles
lat = Math.max(-85, Math.min(85, lat));
lat = Math.max(-max_lat, Math.min(max_lat, lat));
}
}
function onPointerUp() {
isUserInteracting = false;
}
function onSelectStart(event) {
const controller = event.target;
controller.userData.isSelecting = true;
controller.userData.needsInitialPosition = true; // We need to record initial positions in the next frame
}
function onSelectEnd(event) {
const controller = event.target;
controller.userData.isSelecting = false;
controller.userData.initialControllerVectorXZ = null;
controller.userData.needsInitialPosition = false;
}