Files
fblissjr-ComfyUI-QwenImageW…/web/js/qwen_spatial_interface.js
Fred Bliss bf819c6ff5 Add clean spatial token generator implementation
- QwenSpatialTokenGenerator: Pure spatial token generation without template assumptions
- Clean dark-themed spatial interface with canvas drawing
- Supports bounding box, polygon, and object reference tokens
- User-specified labels like [woman] or [green frog]
- No assumptions about user intent
2025-09-05 14:30:14 -05:00

647 lines
26 KiB
JavaScript

/**
* Qwen Spatial Interface
* Clean spatial token generation interface
*/
import { app } from "../../../scripts/app.js";
import { $el } from "../../../scripts/ui.js";
class QwenSpatialInterface {
constructor() {
this.canvas = null;
this.ctx = null;
this.imageCanvas = null;
this.imageCtx = null;
this.regions = [];
this.currentImage = null;
this.isDrawing = false;
this.drawingMode = "bounding_box";
this.polygonPoints = [];
this.imageScale = 1;
this.imageOffset = { x: 0, y: 0 };
}
createInterface(node = null) {
const dialog = $el("div", {
parent: document.body,
style: {
position: "fixed",
top: "5%",
left: "5%",
width: "90%",
height: "90%",
background: "rgba(20, 20, 20, 0.95)",
border: "1px solid rgba(255, 255, 255, 0.2)",
borderRadius: "8px",
zIndex: 10000,
overflow: "hidden",
display: "flex",
flexDirection: "column",
fontFamily: "system-ui, -apple-system, sans-serif",
color: "rgba(255, 255, 255, 0.9)",
backdropFilter: "blur(10px)"
}
});
dialog.innerHTML = `
<!-- Header -->
<div style="
background: rgba(0, 0, 0, 0.3);
padding: 12px 20px;
display: flex;
justify-content: space-between;
align-items: center;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
">
<h2 style="margin: 0; font-size: 18px; font-weight: 500;">Spatial Token Generator</h2>
<button id="closeInterface" style="
padding: 6px 12px;
background: rgba(255, 255, 255, 0.1);
color: white;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
cursor: pointer;
font-size: 12px;
">Close</button>
</div>
<!-- Main Content -->
<div style="flex: 1; display: flex; overflow: hidden;">
<!-- Left Panel -->
<div style="
width: 280px;
background: rgba(0, 0, 0, 0.2);
border-right: 1px solid rgba(255, 255, 255, 0.1);
overflow-y: auto;
padding: 16px;
">
<!-- Image Load -->
<div style="margin-bottom: 20px;">
<h4 style="margin: 0 0 8px 0; font-size: 14px; font-weight: 500;">Load Image</h4>
<input type="file" id="imageUpload" accept="image/*" style="
width: 100%;
padding: 6px;
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
color: white;
font-size: 12px;
">
</div>
<!-- Drawing Mode -->
<div style="margin-bottom: 20px;">
<h4 style="margin: 0 0 8px 0; font-size: 14px; font-weight: 500;">Spatial Type</h4>
<select id="spatialType" style="
width: 100%;
padding: 6px;
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
color: white;
font-size: 12px;
">
<option value="bounding_box">Bounding Box</option>
<option value="polygon">Polygon</option>
<option value="object_reference">Object Reference</option>
</select>
<div style="margin-top: 8px;">
<label style="display: block; margin-bottom: 4px; font-size: 12px; opacity: 0.8;">Label:</label>
<input type="text" id="regionLabel" value="object" style="
width: 100%;
padding: 6px;
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
color: white;
font-size: 12px;
">
</div>
<div id="drawingHelp" style="
margin-top: 8px;
padding: 8px;
background: rgba(255, 255, 255, 0.05);
border-radius: 4px;
font-size: 11px;
opacity: 0.7;
">
Click and drag to create bounding boxes
</div>
</div>
<!-- Regions List -->
<div style="margin-bottom: 20px;">
<h4 style="margin: 0 0 8px 0; font-size: 14px; font-weight: 500;">Regions</h4>
<div id="regionsList" style="
max-height: 150px;
overflow-y: auto;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
padding: 6px;
background: rgba(255, 255, 255, 0.02);
">
<div style="text-align: center; padding: 20px; opacity: 0.5; font-size: 12px;">
No regions created
</div>
</div>
<button id="clearRegions" style="
width: 100%;
padding: 6px;
margin-top: 8px;
background: rgba(255, 255, 255, 0.1);
color: white;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
cursor: pointer;
font-size: 12px;
">Clear All</button>
</div>
<!-- Generate -->
<div>
<button id="generateTokens" style="
width: 100%;
padding: 10px;
background: rgba(0, 120, 255, 0.8);
color: white;
border: 1px solid rgba(0, 120, 255, 0.5);
border-radius: 4px;
cursor: pointer;
font-weight: 500;
">Generate Spatial Tokens</button>
</div>
</div>
<!-- Canvas -->
<div style="
flex: 1;
display: flex;
flex-direction: column;
background: rgba(40, 40, 40, 0.3);
">
<!-- Canvas Controls -->
<div style="
background: rgba(0, 0, 0, 0.2);
padding: 8px;
display: flex;
justify-content: center;
align-items: center;
gap: 12px;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
">
<button id="zoomIn" style="
padding: 4px 8px;
background: rgba(255, 255, 255, 0.1);
color: white;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 3px;
cursor: pointer;
font-size: 11px;
">Zoom +</button>
<span id="zoomLevel" style="font-size: 11px; opacity: 0.7;">100%</span>
<button id="zoomOut" style="
padding: 4px 8px;
background: rgba(255, 255, 255, 0.1);
color: white;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 3px;
cursor: pointer;
font-size: 11px;
">Zoom -</button>
<div style="margin-left: 15px;">
<span id="mouseCoords" style="font-size: 11px; opacity: 0.6;">Mouse: (0, 0)</span>
</div>
</div>
<!-- Drawing Area -->
<div id="canvasContainer" style="
flex: 1;
overflow: hidden;
position: relative;
cursor: crosshair;
">
<canvas id="drawingCanvas" style="
position: absolute;
top: 0;
left: 0;
z-index: 2;
"></canvas>
<canvas id="imageCanvas" style="
position: absolute;
top: 0;
left: 0;
z-index: 1;
"></canvas>
</div>
</div>
<!-- Right Panel -->
<div style="
width: 320px;
background: rgba(0, 0, 0, 0.2);
border-left: 1px solid rgba(255, 255, 255, 0.1);
overflow-y: auto;
padding: 16px;
">
<h4 style="margin: 0 0 12px 0; font-size: 14px; font-weight: 500;">Generated Tokens</h4>
<textarea id="spatialTokensOutput" rows="8" readonly style="
width: 100%;
padding: 8px;
background: rgba(255, 255, 255, 0.05);
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
color: white;
font-family: 'Monaco', 'Menlo', monospace;
font-size: 11px;
resize: vertical;
margin-bottom: 8px;
" placeholder="Generated spatial tokens will appear here..."></textarea>
<div style="display: flex; gap: 6px; margin-bottom: 16px;">
<button id="copyTokens" style="
flex: 1;
padding: 6px;
background: rgba(255, 255, 255, 0.1);
color: white;
border: 1px solid rgba(255, 255, 255, 0.2);
border-radius: 4px;
cursor: pointer;
font-size: 12px;
">Copy</button>
<button id="sendToNode" style="
flex: 1;
padding: 6px;
background: rgba(0, 120, 255, 0.8);
color: white;
border: 1px solid rgba(0, 120, 255, 0.5);
border-radius: 4px;
cursor: pointer;
font-size: 12px;
">Send to Node</button>
</div>
<div id="debugOutput" style="
background: rgba(0, 0, 0, 0.3);
padding: 8px;
border-radius: 4px;
font-family: 'Monaco', 'Menlo', monospace;
font-size: 10px;
max-height: 200px;
overflow-y: auto;
opacity: 0.8;
white-space: pre-wrap;
">Ready to load image...</div>
</div>
</div>
`;
this.setupEventListeners(dialog, node);
this.setupCanvas(dialog);
return dialog;
}
setupEventListeners(dialog, node) {
// Close
dialog.querySelector('#closeInterface').onclick = () => {
document.body.removeChild(dialog);
};
// Image upload
dialog.querySelector('#imageUpload').onchange = (e) => {
const file = e.target.files[0];
if (file) this.loadImage(file, dialog);
};
// Spatial type change
dialog.querySelector('#spatialType').onchange = (e) => {
this.drawingMode = e.target.value;
this.updateDrawingHelp(dialog);
};
// Clear regions
dialog.querySelector('#clearRegions').onclick = () => {
this.regions = [];
this.updateRegionsList(dialog);
this.redrawCanvas();
};
// Generate tokens
dialog.querySelector('#generateTokens').onclick = () => {
this.generateTokens(dialog);
};
// Copy tokens
dialog.querySelector('#copyTokens').onclick = () => {
const tokens = dialog.querySelector('#spatialTokensOutput').value;
navigator.clipboard.writeText(tokens);
};
// Send to node
dialog.querySelector('#sendToNode').onclick = () => {
this.sendToNode(node, dialog);
};
// Zoom controls
dialog.querySelector('#zoomIn').onclick = () => {
this.imageScale = Math.min(this.imageScale * 1.2, 5);
this.updateCanvas(dialog);
};
dialog.querySelector('#zoomOut').onclick = () => {
this.imageScale = Math.max(this.imageScale / 1.2, 0.1);
this.updateCanvas(dialog);
};
}
setupCanvas(dialog) {
const container = dialog.querySelector('#canvasContainer');
this.canvas = dialog.querySelector('#drawingCanvas');
this.ctx = this.canvas.getContext('2d');
this.imageCanvas = dialog.querySelector('#imageCanvas');
this.imageCtx = this.imageCanvas.getContext('2d');
this.resizeCanvases(container.clientWidth, container.clientHeight);
this.setupCanvasEvents(dialog);
new ResizeObserver(() => {
this.resizeCanvases(container.clientWidth, container.clientHeight);
this.updateCanvas(dialog);
}).observe(container);
}
setupCanvasEvents(dialog) {
let startX, startY, isDragging = false;
this.canvas.addEventListener('mousedown', (e) => {
const rect = this.canvas.getBoundingClientRect();
startX = (e.clientX - rect.left) / this.imageScale;
startY = (e.clientY - rect.top) / this.imageScale;
if (this.drawingMode === 'polygon') {
this.polygonPoints.push([Math.round(startX), Math.round(startY)]);
this.redrawCanvas();
if (e.detail === 2 && this.polygonPoints.length >= 3) {
this.finishPolygon(dialog);
}
} else {
isDragging = true;
}
});
this.canvas.addEventListener('mousemove', (e) => {
const rect = this.canvas.getBoundingClientRect();
const currentX = (e.clientX - rect.left) / this.imageScale;
const currentY = (e.clientY - rect.top) / this.imageScale;
dialog.querySelector('#mouseCoords').textContent = `Mouse: (${Math.round(currentX)}, ${Math.round(currentY)})`;
if (isDragging && this.drawingMode === 'bounding_box') {
this.redrawCanvas();
this.drawPreviewBox(startX, startY, currentX, currentY);
}
});
this.canvas.addEventListener('mouseup', (e) => {
if (!isDragging) return;
const rect = this.canvas.getBoundingClientRect();
const endX = (e.clientX - rect.left) / this.imageScale;
const endY = (e.clientY - rect.top) / this.imageScale;
if (this.drawingMode === 'bounding_box') {
if (Math.abs(endX - startX) > 5 && Math.abs(endY - startY) > 5) {
this.addBoundingBox(startX, startY, endX, endY, dialog);
}
} else if (this.drawingMode === 'object_reference') {
this.addObjectReference(startX, startY, dialog);
}
isDragging = false;
});
}
loadImage(file, dialog) {
const reader = new FileReader();
reader.onload = (e) => {
const img = new Image();
img.onload = () => {
this.currentImage = img;
this.imageScale = 1;
this.regions = [];
this.updateCanvas(dialog);
this.updateRegionsList(dialog);
this.updateDebug(`Image loaded: ${img.width}x${img.height}px`, dialog);
};
img.src = e.target.result;
};
reader.readAsDataURL(file);
}
updateCanvas(dialog) {
if (!this.currentImage) return;
this.imageCtx.clearRect(0, 0, this.imageCanvas.width, this.imageCanvas.height);
const imgWidth = this.currentImage.width * this.imageScale;
const imgHeight = this.currentImage.height * this.imageScale;
this.imageCtx.drawImage(this.currentImage, 0, 0, imgWidth, imgHeight);
dialog.querySelector('#zoomLevel').textContent = `${Math.round(this.imageScale * 100)}%`;
this.redrawCanvas();
}
redrawCanvas() {
if (!this.canvas) return;
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
this.regions.forEach((region, index) => {
const color = ['#ff4444', '#44ff44', '#4444ff', '#ffff44'][index % 4];
this.drawRegion(region, color);
});
if (this.drawingMode === 'polygon' && this.polygonPoints.length > 0) {
this.drawPolygonInProgress();
}
}
drawRegion(region, color) {
this.ctx.strokeStyle = color;
this.ctx.fillStyle = color + '20';
this.ctx.lineWidth = 2;
if (region.type === 'bounding_box') {
const [x1, y1, x2, y2] = region.coords;
const drawX1 = x1 * this.imageScale;
const drawY1 = y1 * this.imageScale;
const drawWidth = (x2 - x1) * this.imageScale;
const drawHeight = (y2 - y1) * this.imageScale;
this.ctx.strokeRect(drawX1, drawY1, drawWidth, drawHeight);
this.ctx.fillRect(drawX1, drawY1, drawWidth, drawHeight);
this.ctx.fillStyle = color;
this.ctx.font = '12px system-ui';
this.ctx.fillText(region.label, drawX1, drawY1 - 5);
}
}
drawPreviewBox(x1, y1, x2, y2) {
this.ctx.strokeStyle = 'rgba(255, 255, 255, 0.8)';
this.ctx.setLineDash([5, 5]);
this.ctx.lineWidth = 1;
const drawX1 = x1 * this.imageScale;
const drawY1 = y1 * this.imageScale;
const drawWidth = (x2 - x1) * this.imageScale;
const drawHeight = (y2 - y1) * this.imageScale;
this.ctx.strokeRect(drawX1, drawY1, drawWidth, drawHeight);
this.ctx.setLineDash([]);
}
addBoundingBox(x1, y1, x2, y2, dialog) {
const label = dialog.querySelector('#regionLabel').value || 'object';
const region = {
type: 'bounding_box',
label: label,
coords: [
Math.min(x1, x2),
Math.min(y1, y2),
Math.max(x1, x2),
Math.max(y1, y2)
]
};
this.regions.push(region);
this.updateRegionsList(dialog);
this.redrawCanvas();
this.updateDebug(`Added bounding box: ${label}`, dialog);
}
updateRegionsList(dialog) {
const listContainer = dialog.querySelector('#regionsList');
if (this.regions.length === 0) {
listContainer.innerHTML = '<div style="text-align: center; padding: 20px; opacity: 0.5; font-size: 12px;">No regions created</div>';
return;
}
const listHTML = this.regions.map((region, index) => {
const color = ['#ff4444', '#44ff44', '#4444ff', '#ffff44'][index % 4];
return `
<div style="
padding: 6px;
margin-bottom: 4px;
border: 1px solid ${color}40;
border-radius: 3px;
background: ${color}10;
font-size: 12px;
">
<div style="color: ${color}; font-weight: 500;">${region.label}</div>
<div style="opacity: 0.7; font-size: 10px;">${region.type}</div>
</div>
`;
}).join('');
listContainer.innerHTML = listHTML;
}
generateTokens(dialog) {
if (!this.currentImage || this.regions.length === 0) {
this.updateDebug("No regions to generate tokens from", dialog);
return;
}
const tokens = this.regions.map(region => {
const [x1, y1, x2, y2] = region.coords;
const normX1 = x1 / this.currentImage.width;
const normY1 = y1 / this.currentImage.height;
const normX2 = x2 / this.currentImage.width;
const normY2 = y2 / this.currentImage.height;
return `<|object_ref_start|>${region.label}<|object_ref_end|> at <|box_start|>${normX1.toFixed(3)},${normY1.toFixed(3)},${normX2.toFixed(3)},${normY2.toFixed(3)}<|box_end|>`;
});
const spatialTokens = tokens.join(' ');
dialog.querySelector('#spatialTokensOutput').value = spatialTokens;
this.updateDebug(`Generated ${tokens.length} spatial tokens`, dialog);
}
sendToNode(node, dialog) {
const tokens = dialog.querySelector('#spatialTokensOutput').value;
if (!tokens) return;
// Try to find a text widget in the connected node
if (node && node.widgets) {
const textWidget = node.widgets.find(w =>
w.name === 'text' || w.name === 'prompt' || w.name === 'coordinates'
);
if (textWidget) {
textWidget.value = tokens;
node.setDirtyCanvas(true, true);
this.updateDebug(`Sent tokens to node widget: ${textWidget.name}`, dialog);
} else {
this.updateDebug("No compatible text widget found in node", dialog);
}
}
}
updateDrawingHelp(dialog) {
const help = {
'bounding_box': 'Click and drag to create bounding boxes',
'polygon': 'Click to add points. Double-click to finish',
'object_reference': 'Click on objects to mark them'
};
dialog.querySelector('#drawingHelp').textContent = help[this.drawingMode];
}
updateDebug(message, dialog) {
const debugOutput = dialog.querySelector('#debugOutput');
const timestamp = new Date().toLocaleTimeString();
debugOutput.textContent += `${timestamp}: ${message}\n`;
debugOutput.scrollTop = debugOutput.scrollHeight;
}
resizeCanvases(width, height) {
if (this.canvas) {
this.canvas.width = width;
this.canvas.height = height;
}
if (this.imageCanvas) {
this.imageCanvas.width = width;
this.imageCanvas.height = height;
}
}
}
// Register extension
app.registerExtension({
name: "Comfy.QwenSpatialInterface",
async beforeRegisterNodeDef(nodeType, nodeData, app) {
if (nodeData.name === "QwenSpatialTokenGenerator") {
const onNodeCreated = nodeType.prototype.onNodeCreated;
nodeType.prototype.onNodeCreated = function() {
const ret = onNodeCreated ? onNodeCreated.apply(this, arguments) : undefined;
const node = this;
setTimeout(() => {
const interfaceBtn = node.addWidget("button", "Open Spatial Interface", "interface", () => {
const spatialInterface = new QwenSpatialInterface();
spatialInterface.createInterface(node);
});
}, 10);
return ret;
};
}
}
});