Files
VrchStudio-comfyui-web-viewer/web/comfyui/audio_mic_loader_node.js
T

715 lines
27 KiB
JavaScript

import { app } from "../../scripts/app.js";
import { api } from "../../scripts/api.js";
import {
hideWidget,
showWidget,
createMicrophoneControls
} from "./node_utils.js";
app.registerExtension({
name: "vrch.MicLoader",
async nodeCreated(node) {
if (node.comfyClass === "VrchMicLoaderNode") {
// Get required widgets
const deviceIdWidget = node.widgets.find(w => w.name === "device_id");
const nameWidget = node.widgets.find(w => w.name === "name");
const sensitivityWidget = node.widgets.find(w => w.name === "sensitivity");
const frameSizeWidget = node.widgets.find(w => w.name === "frame_size");
const sampleRateWidget = node.widgets.find(w => w.name === "sample_rate");
const enablePreviewWidget = node.widgets.find(w => w.name === "enable_preview");
const rawDataWidget = node.widgets.find(w => w.name === "raw_data");
const debugWidget = node.widgets.find(w => w.name === "debug");
// State variables
let audioContext = null;
let analyser = null;
let microphone = null;
let gainNode = null;
let animationFrameId = null;
let isCapturing = false;
let isMuted = false;
let suppressMuteCallback = false;
// Hide technical widgets from the UI
if (deviceIdWidget) {
hideWidget(node, deviceIdWidget);
}
if (rawDataWidget) {
hideWidget(node, rawDataWidget);
}
// Function to update widget visibility
const updateWidgetVisibility = () => {
if (debugWidget && rawDataWidget) {
if (debugWidget.value) {
showWidget(node, rawDataWidget);
} else {
hideWidget(node, rawDataWidget);
}
}
};
if (debugWidget) {
debugWidget.callback = (value) => {
updateWidgetVisibility();
};
}
if (enablePreviewWidget) {
enablePreviewWidget.callback = (value) => {
updateVisualizerVisibility();
};
}
// Create visualizer elements
const visualizerContainer = document.createElement("div");
visualizerContainer.classList.add("mic-visualizer-container");
// Create waveform canvas
const waveformCanvas = document.createElement("canvas");
waveformCanvas.width = 300;
waveformCanvas.height = 100;
waveformCanvas.classList.add("mic-canvas");
// Create spectrum canvas
const spectrumCanvas = document.createElement("canvas");
spectrumCanvas.width = 300;
spectrumCanvas.height = 100;
spectrumCanvas.classList.add("mic-canvas");
// Create volume meter container
const volumeMeterContainer = document.createElement("div");
volumeMeterContainer.classList.add("mic-volume-meter-container");
const volumeMeterFill = document.createElement("div");
volumeMeterFill.classList.add("mic-volume-meter-fill");
volumeMeterContainer.appendChild(volumeMeterFill);
let micControls;
let statusIndicator = null;
const updateStatusIndicator = (status) => {
if (micControls) {
micControls.setStatus(status);
}
};
const updateMuteState = (mute, { syncControl = false } = {}) => {
isMuted = mute;
if (gainNode) {
gainNode.gain.value = mute ? 0 : 1;
}
if (mute) {
volumeMeterFill.style.width = "0%";
}
if (syncControl && micControls) {
suppressMuteCallback = true;
micControls.setMuted(mute);
suppressMuteCallback = false;
}
if (statusIndicator) {
const text = statusIndicator.textContent;
if (text === "Status: Ready" || text === "Status: Muted") {
updateStatusIndicator(mute ? "Muted" : "Ready");
}
}
};
const handleDeviceSelection = (deviceId) => {
if (deviceIdWidget) {
deviceIdWidget.value = deviceId || "";
}
if (nameWidget) {
const selectedDevice = micControls ? micControls.devices().find(d => d.deviceId === deviceId) : null;
if (selectedDevice && selectedDevice.label) {
nameWidget.value = selectedDevice.label;
} else if (!deviceId) {
nameWidget.value = "";
}
}
if (deviceId) {
updateStatusIndicator(isMuted ? "Muted" : "Ready");
} else {
updateStatusIndicator("Ready");
}
};
micControls = createMicrophoneControls({
onDeviceChange: handleDeviceSelection,
onMuteChange: (muted) => {
if (suppressMuteCallback) {
return;
}
updateMuteState(muted);
}
});
const deviceSelect = micControls.selectEl;
const reloadButton = micControls.reloadButton;
const muteButton = micControls.muteButton;
statusIndicator = micControls.statusLabel;
handleDeviceSelection(micControls.getSelection());
visualizerContainer.appendChild(waveformCanvas);
visualizerContainer.appendChild(spectrumCanvas);
visualizerContainer.appendChild(volumeMeterContainer);
visualizerContainer.appendChild(micControls.container);
// Function to update visualizer visibility (only visual elements)
const updateVisualizerVisibility = () => {
if (enablePreviewWidget) {
// Only hide/show the canvas elements and volume meter, keep controls visible
const displayValue = enablePreviewWidget.value ? "block" : "none";
waveformCanvas.style.display = displayValue;
spectrumCanvas.style.display = displayValue;
volumeMeterContainer.style.display = displayValue;
}
};
// Add the visualizer container as a DOM widget
const widget = node.addDOMWidget("mic_visualizer_widget", "Microphone", visualizerContainer);
// Override the computeSize method to dynamically adjust height based on enable_preview
widget.computeSize = function(width) {
const baseHeight = 100;
const visualizerHeight = 280;
// Check if preview is enabled
const isPreviewEnabled = enablePreviewWidget ? enablePreviewWidget.value : true;
const totalHeight = baseHeight + (isPreviewEnabled ? visualizerHeight : 0);
return [width, totalHeight];
};
// Function to enumerate and list available audio devices
const listAudioDevices = async ({ requestAccess = false } = {}) => {
updateStatusIndicator("Loading");
try {
const deviceList = await micControls.refreshDevices({
requestAccess,
logPrefix: "[VrchMicLoaderNode]"
});
const savedDeviceId = deviceIdWidget ? deviceIdWidget.value : "";
if (savedDeviceId && !micControls.getSelection()) {
micControls.setSelection(savedDeviceId);
}
handleDeviceSelection(micControls.getSelection());
updateMuteState(isMuted, { syncControl: true });
return true; // Indicate successful completion
} catch (error) {
console.error("[VrchMicLoaderNode] Error listing audio devices:", error);
updateStatusIndicator("Error");
return false; // Indicate error
}
};
// Function to request microphone access and start capturing
const startCapturing = async () => {
try {
// Stop any existing capture
stopCapturing();
const selectedDeviceId = micControls.getSelection();
if (!selectedDeviceId) {
updateStatusIndicator("No device");
return;
}
// Create audio context
audioContext = new (window.AudioContext || window.webkitAudioContext)({
sampleRate: parseInt(sampleRateWidget.value, 10)
});
// Request microphone access
const constraints = {
audio: {
deviceId: selectedDeviceId ? { exact: selectedDeviceId } : undefined,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: false
}
};
const stream = await navigator.mediaDevices.getUserMedia(constraints);
// Set up audio processing pipeline
microphone = audioContext.createMediaStreamSource(stream);
analyser = audioContext.createAnalyser();
gainNode = audioContext.createGain();
// Configure analyser
analyser.fftSize = parseInt(frameSizeWidget.value, 10) * 2; // FFT size must be 2x frame size
analyser.smoothingTimeConstant = 0.8;
// Connect nodes: microphone -> gain -> analyser
microphone.connect(gainNode);
gainNode.connect(analyser);
// Update mute state
updateMuteState(isMuted, { syncControl: true });
// Get the selected device info
const selectedDevice = micControls.devices().find(d => d.deviceId === selectedDeviceId);
// Update widgets with device info
deviceIdWidget.value = selectedDeviceId;
if (selectedDevice && selectedDevice.label) {
nameWidget.value = selectedDevice.label;
}
// Start visualization and data collection
isCapturing = true;
updateStatusIndicator(isMuted ? "Muted" : "Ready");
startVisualization();
} catch (error) {
console.error("Error accessing microphone:", error);
updateStatusIndicator("Error");
}
};
// Function to stop capturing
const stopCapturing = () => {
if (animationFrameId) {
cancelAnimationFrame(animationFrameId);
animationFrameId = null;
}
if (microphone && microphone.mediaStream) {
// Get all tracks from the stream
const tracks = microphone.mediaStream.getTracks();
tracks.forEach(track => track.stop());
microphone = null;
}
if (gainNode) {
gainNode.disconnect();
gainNode = null;
}
if (analyser) {
analyser.disconnect();
analyser = null;
}
if (audioContext) {
audioContext.close();
audioContext = null;
}
isCapturing = false;
updateStatusIndicator("Disconnected");
// Clear visualizations
clearCanvas(waveformCanvas);
clearCanvas(spectrumCanvas);
volumeMeterFill.style.width = "0%";
};
// Function to start the visualization loop
const startVisualization = () => {
if (!analyser) return;
const waveformCtx = waveformCanvas.getContext("2d");
const spectrumCtx = spectrumCanvas.getContext("2d");
const frameSize = parseInt(frameSizeWidget.value, 10);
const timeData = new Uint8Array(analyser.fftSize);
const frequencyData = new Uint8Array(analyser.frequencyBinCount);
const updateVisualization = () => {
if (!analyser || !isCapturing) return;
// Get time domain data (waveform)
analyser.getByteTimeDomainData(timeData);
// Get frequency domain data (spectrum)
analyser.getByteFrequencyData(frequencyData);
// Calculate RMS (volume)
let rms = 0;
for (let i = 0; i < timeData.length; i++) {
const amplitude = (timeData[i] - 128) / 128;
rms += amplitude * amplitude;
}
rms = Math.sqrt(rms / timeData.length);
// Calculate zero-crossing rate (ZCR)
let zcr = 0;
for (let i = 1; i < timeData.length; i++) {
if ((timeData[i - 1] < 128 && timeData[i] >= 128) ||
(timeData[i - 1] >= 128 && timeData[i] < 128)) {
zcr++;
}
}
zcr = zcr / (timeData.length - 1);
// Find peak amplitude
let peak = 0;
for (let i = 0; i < timeData.length; i++) {
const amplitude = Math.abs((timeData[i] - 128) / 128);
if (amplitude > peak) peak = amplitude;
}
// Apply sensitivity
const sensitivity = sensitivityWidget ? sensitivityWidget.value : 0.5;
const adjustedRms = Math.min(1.0, rms * (1.0 / sensitivity));
// Determine if microphone is active
const threshold = 0.05 * (1.0 - sensitivity); // Lower sensitivity = higher threshold
const isActive = adjustedRms > threshold;
// Create normalized waveform and spectrum arrays
const waveformArray = Array(128).fill(0);
const spectrumArray = Array(128).fill(0);
// Create normalized waveform array (128 values)
const waveStep = Math.max(1, Math.floor(timeData.length / 128));
for (let i = 0; i < 128; i++) {
const dataIdx = Math.min(i * waveStep, timeData.length - 1);
waveformArray[i] = (timeData[dataIdx] - 128) / 128.0; // -1.0 to 1.0
}
// Create normalized spectrum array (128 values)
const specStep = Math.max(1, Math.floor(frequencyData.length / 128));
for (let i = 0; i < 128; i++) {
const dataIdx = Math.min(i * specStep, frequencyData.length - 1);
spectrumArray[i] = frequencyData[dataIdx] / 255.0; // 0.0 to 1.0
}
// Draw waveform
drawWaveform(waveformCtx, timeData);
// Draw spectrum
drawSpectrum(spectrumCtx, frequencyData);
// Update volume meter
volumeMeterFill.style.width = `${adjustedRms * 100}%`;
// Update volume meter color based on level
if (adjustedRms < 0.3) {
volumeMeterFill.style.backgroundColor = "#4CAF50"; // Green
} else if (adjustedRms < 0.7) {
volumeMeterFill.style.backgroundColor = "#FFC107"; // Yellow
} else {
volumeMeterFill.style.backgroundColor = "#F44336"; // Red
}
// Prepare and send data to Python backend
if (rawDataWidget) {
const micData = {
device_id: deviceIdWidget.value,
name: nameWidget.value,
sr: parseInt(sampleRateWidget.value, 10),
ch: 1, // Mono for simplicity
hop: parseInt(frameSizeWidget.value, 10),
waveform: waveformArray,
spectrum: spectrumArray,
volume: adjustedRms,
is_active: isActive,
rms: rms,
zcr: zcr,
peak: peak
};
rawDataWidget.value = JSON.stringify(micData);
}
// Continue animation loop
animationFrameId = requestAnimationFrame(updateVisualization);
};
// Start the animation loop
updateVisualization();
};
// Function to draw waveform on canvas
const drawWaveform = (ctx, data) => {
const width = ctx.canvas.width;
const height = ctx.canvas.height;
// Clear canvas
ctx.clearRect(0, 0, width, height);
// Draw background
ctx.fillStyle = "#111";
ctx.fillRect(0, 0, width, height);
// Draw center line
ctx.beginPath();
ctx.strokeStyle = "#333";
ctx.moveTo(0, height / 2);
ctx.lineTo(width, height / 2);
ctx.stroke();
// Draw waveform - changed from green to grayscale
ctx.beginPath();
ctx.strokeStyle = "#CCCCCC"; // Light gray instead of green
ctx.lineWidth = 2;
const sliceWidth = width / data.length;
let x = 0;
for (let i = 0; i < data.length; i++) {
const v = data[i] / 255;
const y = v * height;
if (i === 0) {
ctx.moveTo(x, y);
} else {
ctx.lineTo(x, y);
}
x += sliceWidth;
}
ctx.stroke();
};
// Function to draw frequency spectrum on canvas
const drawSpectrum = (ctx, data) => {
const width = ctx.canvas.width;
const height = ctx.canvas.height;
// Clear canvas
ctx.clearRect(0, 0, width, height);
// Draw background
ctx.fillStyle = "#111";
ctx.fillRect(0, 0, width, height);
// Calculate bar width
const barCount = Math.min(128, data.length);
const barWidth = width / barCount;
// Draw frequency bars
for (let i = 0; i < barCount; i++) {
// Get a subset of the data to avoid too many bars
const dataIndex = Math.floor(i * data.length / barCount);
const value = data[dataIndex];
const percent = value / 255;
// Calculate bar height based on full height
const barHeight = percent * height;
const x = i * barWidth;
const y = height - barHeight;
// Restore colorful spectrum
const hue = i / barCount * 120; // 0-120 gives a range from red to green
ctx.fillStyle = `hsl(${hue}, 100%, 50%)`;
ctx.fillRect(x, y, barWidth, barHeight); // Removed -1 to eliminate gaps
}
};
// Function to clear canvas
const clearCanvas = (canvas) => {
const ctx = canvas.getContext("2d");
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.fillStyle = "#111";
ctx.fillRect(0, 0, canvas.width, canvas.height);
};
// Set up event listeners
deviceSelect.addEventListener("change", () => {
if (micControls.getSelection()) {
startCapturing();
} else {
stopCapturing();
}
});
reloadButton.addEventListener("click", async () => {
// First stop any existing capture
stopCapturing();
// Then reload device list
const devicesLoaded = await listAudioDevices({ requestAccess: true });
// Automatically restart if a device is selected
if (devicesLoaded && micControls.getSelection()) {
startCapturing();
}
});
// Set a delay initialization for the widget visibility
setTimeout(async () => {
// Initial setup
updateWidgetVisibility();
updateVisualizerVisibility();
// Store the previously selected deviceId for recovery
const savedDeviceId = deviceIdWidget.value;
// Request microphone permissions and list devices
await navigator.mediaDevices.getUserMedia({ audio: true }).catch(err => {
console.warn("Microphone permission request failed:", err);
});
// List audio devices with proper permissions
const devicesLoaded = await listAudioDevices();
// Try to restore the previously selected device
if (devicesLoaded) {
if (savedDeviceId && savedDeviceId !== "") {
micControls.setSelection(savedDeviceId);
handleDeviceSelection(micControls.getSelection());
if (micControls.getSelection()) {
console.log("Restored previously selected microphone:", savedDeviceId);
setTimeout(() => {
startCapturing();
}, 100);
}
} else {
updateStatusIndicator("Ready"); // No device selected
}
}
}, 1000);
// Cleanup when node is removed
const onRemoved = node.onRemoved;
node.onRemoved = function() {
stopCapturing();
if (onRemoved) {
onRemoved.apply(this, arguments);
}
};
}
}
});
// Add custom styles
const style = document.createElement("style");
style.textContent = `
.mic-visualizer-container {
width: 100%;
display: flex;
flex-direction: column;
gap: 10px;
margin-top: 10px;
min-width: 0;
box-sizing: border-box;
}
.mic-canvas {
width: 100%;
height: 100px;
background-color: #111;
border-radius: 4px;
flex-shrink: 0;
min-width: 0;
box-sizing: border-box;
}
.mic-volume-meter-container {
width: 100%;
height: 20px;
background-color: #111;
border-radius: 4px;
position: relative;
overflow: hidden;
flex-shrink: 0;
min-width: 0;
}
.mic-volume-meter-fill {
height: 100%;
background-color: #4CAF50;
width: 0%;
transition: width 0.1s ease;
}
.mic-controls-container {
display: flex;
flex-direction: column;
gap: 8px;
margin-top: 10px;
}
.mic-device-row {
display: flex;
width: 100%;
}
.mic-button-row {
display: flex;
justify-content: center;
gap: 10px;
margin-top: 5px;
}
.mic-device-select {
width: 100%;
padding: 5px;
border-radius: 4px;
}
.vrch-mic-reload-button {
background-color: #4CAF50;
color: white;
height: 30px;
font-size: 14px;
font-weight: bold;
border: none;
border-radius: 4px;
padding: 5px 10px;
margin: 0 5px;
cursor: pointer;
transition: background-color 0.3s;
min-width: 35px;
display: flex;
align-items: center;
justify-content: center;
}
.vrch-mic-mute-button {
background-color: #4f9cff;
color: white;
height: 30px;
font-size: 14px;
font-weight: bold;
border: none;
border-radius: 4px;
padding: 5px 10px;
margin: 0 5px;
cursor: pointer;
transition: background-color 0.3s;
min-width: 35px;
display: flex;
align-items: center;
justify-content: center;
}
.vrch-mic-reload-button:hover,
.vrch-mic-mute-button:hover {
filter: brightness(1.1);
}
.vrch-mic-status-container {
width: 100%;
display: flex;
justify-content: center;
margin-top: 0px;
}
.vrch-mic-status-label {
font-size: 14px;
color: #999;
text-align: center;
}
.vrch-mic-speaker-icon {
font-size: 16px;
}
`;
document.head.appendChild(style);