nodes/benfinklea-ComfyUI-ModelSync
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Unify lookup and check into single concurrent operation

Browse Source 
- Merged "Lookup Sources" and "Check Updates" into one button
- Now processes models concurrently (5 at a time by default)
- Single click: hashes files, finds sources, checks for updates
- 10 models processed in ~2 seconds vs sequential approach

Simpler UX with faster performance through asyncio.gather().

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Ben Finklea
2026-01-25 13:53:38 -06:00
co-authored by Claude Opus 4.5
parent 69197802b6
commit f807d1b927
2 changed files with 124 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/modelsync-panel.js
+26 -92
View File
@@ -177,9 +177,7 @@ class ModelSyncPanel {
<option value="auto">Auto-Update Enabled</option>
</select>
<button class="modelsync-btn" data-action="scan">Scan</button>
<button class="modelsync-btn primary" data-action="lookup">Lookup Sources</button>
<button class="modelsync-btn" data-action="stop-lookup" style="display:none;background:#c00;">Stop</button>
<button class="modelsync-btn" data-action="check">Check Updates</button>
<button class="modelsync-btn primary" data-action="check">Check Updates</button>
</div>
<div class="modelsync-models-list"></div>
</div>
@@ -280,12 +278,6 @@ class ModelSyncPanel {
case "scan":
await this.scanModels();
break;
case "lookup":
await this.lookupSources();
break;
case "stop-lookup":
this.stopLookup();
break;
case "check":
await this.checkUpdates();
break;
@@ -313,99 +305,41 @@ class ModelSyncPanel {
}
}
async lookupSources() {
this.lookupRunning = true;
let foundCount = 0;
let notFoundCount = 0;
let processedCount = 0;
// Show stop button, hide lookup button
const lookupBtn = this.element?.querySelector('[data-action="lookup"]');
const stopBtn = this.element?.querySelector('[data-action="stop-lookup"]');
if (lookupBtn) lookupBtn.style.display = 'none';
if (stopBtn) stopBtn.style.display = 'inline-block';
const processNext = async () => {
if (!this.lookupRunning) {
if (lookupBtn) lookupBtn.style.display = 'inline-block';
if (stopBtn) stopBtn.style.display = 'none';
this.setStatus(`Lookup stopped. Found ${foundCount}, not found ${notFoundCount}.`);
await this.refreshData();
return;
}
try {
const res = await fetch("/modelsync/lookup-sources", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({}),
});
const data = await res.json();
if (data.status === "complete") {
this.lookupRunning = false;
if (lookupBtn) lookupBtn.style.display = 'inline-block';
if (stopBtn) stopBtn.style.display = 'none';
this.setStatus(`Lookup complete. Found ${foundCount} on CivitAI, ${notFoundCount} not found.`);
await this.refreshData();
return;
}
if (data.status === "found") {
foundCount++;
processedCount++;
} else if (data.status === "hint") {
foundCount++; // Count hints as partial success
processedCount++;
} else if (data.status === "not_found") {
notFoundCount++;
processedCount++;
} else if (data.status === "hashing") {
// Just hashed, will lookup next call
}
const progress = data.total > 0 ? Math.round(((data.total - data.remaining) / data.total) * 100) : 0;
this.setStatus(`[${progress}%] ${data.message} (${foundCount} found, ${data.remaining} remaining)`);
// Update UI periodically
if (processedCount % 5 === 0) {
await this.refreshData();
}
// Continue to next model
setTimeout(processNext, 100);
} catch (error) {
this.lookupRunning = false;
if (lookupBtn) lookupBtn.style.display = 'inline-block';
if (stopBtn) stopBtn.style.display = 'none';
this.setStatus("Lookup error: " + error.message);
await this.refreshData();
}
};
this.setStatus("Starting source lookup...");
processNext();
}
stopLookup() {
this.lookupRunning = false;
this.setStatus("Stopping lookup...");
}
async checkUpdates() {
this.setStatus("Checking for updates...");
this.setStatus("Checking updates (hashing, finding sources, checking versions)...");
// Disable button during check
const checkBtn = this.element?.querySelector('[data-action="check"]');
if (checkBtn) {
checkBtn.disabled = true;
checkBtn.textContent = "Checking...";
}
try {
const res = await fetch("/modelsync/check", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({}),
body: JSON.stringify({ concurrent: 5 }),
});
const data = await res.json();
this.setStatus(`Checked ${data.checked} models. Updates available: ${data.updates_available}`);
const parts = [];
if (data.hashed > 0) parts.push(`${data.hashed} hashed`);
if (data.sources_found > 0) parts.push(`${data.sources_found} sources found`);
if (data.checked > 0) parts.push(`${data.checked} checked`);
if (data.updates_available > 0) parts.push(`${data.updates_available} updates!`);
if (data.errors > 0) parts.push(`${data.errors} errors`);
const summary = parts.length > 0 ? parts.join(", ") : "No models to process";
this.setStatus(`Done: ${summary}`);
await this.refreshData();
} catch (error) {
this.setStatus("Check failed: " + error.message);
} finally {
if (checkBtn) {
checkBtn.disabled = false;
checkBtn.textContent = "Check Updates";
}
}
}
server.py
+98 -154
View File
@@ -71,7 +71,6 @@ class ModelSyncAPI:
app.router.add_post("/modelsync/check", self.check_updates)
app.router.add_post("/modelsync/download/{id}", self.download_update)
app.router.add_post("/modelsync/lookup-sources", self.lookup_sources)
app.router.add_get("/modelsync/config", self.get_config)
app.router.add_post("/modelsync/config", self.set_config)
@@ -208,50 +207,115 @@ class ModelSyncAPI:
# Version check endpoints
async def check_updates(self, request: web.Request) -> web.Response:
"""Check models for updates."""
"""
Unified check: finds sources and checks for updates concurrently.
Processes models in batches for efficiency.
"""
import asyncio
data = await request.json() if request.body_exists else {}
model_ids = data.get("model_ids")
concurrent_limit = data.get("concurrent", 5)
all_models = self.store.get_all_models()
if model_ids:
models = [self.store.get_model(mid) for mid in model_ids]
models = [m for m in models if m and m.get("source_url")]
else:
models = self.store.get_all_models()
models = [m for m in models if m.get("source_url")]
all_models = [m for m in all_models if m["id"] in model_ids]
if not models:
return web.json_response({"checked": 0, "updates_available": 0})
if not all_models:
return web.json_response({
"total": 0, "checked": 0, "sources_found": 0,
"updates_available": 0, "status": "complete"
})
results = await self.tracker.batch_check(models)
updates_found = 0
# Track already-checked source types
checked_types = {"not_found", "civitai", "huggingface", "github", "hint"}
for model in models:
model_id = model["id"]
if model_id in results:
remote_info = results[model_id]
stats = {
"total": len(all_models),
"hashed": 0,
"sources_found": 0,
"checked": 0,
"updates_available": 0,
"errors": 0,
}
is_update = self.tracker.is_update_available(
local_hash=model.get("file_hash"),
local_size=model.get("file_size"),
remote_hash=remote_info.get("hash"),
remote_size=remote_info.get("size"),
)
semaphore = asyncio.Semaphore(concurrent_limit)
self.store.update_model(model_id, {
"remote_hash": remote_info.get("hash"),
"remote_size": remote_info.get("size"),
"remote_modified": remote_info.get("modified"),
"last_checked": datetime.now().isoformat(),
"update_available": 1 if is_update else 0,
})
async def process_model(model):
"""Process a single model: hash → find source → check update."""
async with semaphore:
model_id = model["id"]
filename = model["filename"]
if is_update:
updates_found += 1
try:
# Step 1: Compute hash if missing
if not model.get("file_hash"):
try:
file_hash = compute_sha256_chunked(model["path"])
self.store.update_model(model_id, {"file_hash": file_hash})
model["file_hash"] = file_hash
stats["hashed"] += 1
except Exception as e:
print(f"[ModelSync] Hash error {filename}: {e}")
stats["errors"] += 1
return
return web.json_response({
"checked": len(models),
"updates_available": updates_found,
})
# Step 2: Find source if missing
if not model.get("source_url") and model.get("source_type") not in checked_types:
info = await self.tracker.smart_lookup(filename, model.get("file_hash"))
if info and info.get("source_url"):
self.store.update_model(model_id, {
"source_url": info.get("source_url"),
"source_type": info.get("source_type", "unknown"),
})
model["source_url"] = info.get("source_url")
model["source_type"] = info.get("source_type")
stats["sources_found"] += 1
elif info and info.get("source_hint"):
self.store.update_model(model_id, {
"source_type": info.get("source_type", "hint"),
"source_url": info.get("source_hint"),
})
model["source_url"] = info.get("source_hint")
stats["sources_found"] += 1
else:
self.store.update_model(model_id, {"source_type": "not_found"})
# Step 3: Check for updates if we have a source URL
source_url = model.get("source_url")
if source_url and source_url.startswith("http"):
remote_info = await self.tracker.check_version(source_url)
if remote_info:
is_update = self.tracker.is_update_available(
local_hash=model.get("file_hash"),
local_size=model.get("file_size"),
remote_hash=remote_info.get("hash"),
remote_size=remote_info.get("size"),
)
self.store.update_model(model_id, {
"remote_hash": remote_info.get("hash"),
"remote_size": remote_info.get("size"),
"remote_modified": remote_info.get("modified"),
"last_checked": datetime.now().isoformat(),
"update_available": 1 if is_update else 0,
})
stats["checked"] += 1
if is_update:
stats["updates_available"] += 1
except Exception as e:
print(f"[ModelSync] Error processing {filename}: {e}")
stats["errors"] += 1
# Process all models concurrently
await asyncio.gather(*[process_model(m) for m in all_models])
stats["status"] = "complete"
return web.json_response(stats)
async def download_update(self, request: web.Request) -> web.Response:
"""Download update for a specific model."""
@@ -271,126 +335,6 @@ class ModelSyncAPI:
"model_id": model_id,
})
async def lookup_sources(self, request: web.Request) -> web.Response:
"""
Look up source URL for ONE model at a time by querying CivitAI with file hash.
Call repeatedly to process all models with progress feedback.
"""
data = await request.json() if request.body_exists else {}
# Get all models that haven't been looked up yet
# Exclude: models with source_url, or source_type indicating already checked
all_models = self.store.get_all_models()
checked_types = {"not_found", "civitai", "huggingface", "github", "hint"}
models_to_lookup = [
m for m in all_models
if not m.get("source_url") and m.get("source_type") not in checked_types
]
if not models_to_lookup:
return web.json_response({
"status": "complete",
"message": "All models already have sources",
"remaining": 0,
"total": len(all_models),
})
# Find next model to process (prioritize ones with hashes)
model = None
for m in models_to_lookup:
if m.get("file_hash"):
model = m
break
# If no model has hash, pick first one to hash
if not model:
model = models_to_lookup[0]
# Step 1: Compute hash if needed
if not model.get("file_hash"):
try:
print(f"[ModelSync] Hashing {model['filename']}...")
file_hash = compute_sha256_chunked(model["path"])
self.store.update_model(model["id"], {"file_hash": file_hash})
model["file_hash"] = file_hash
return web.json_response({
"status": "hashing",
"message": f"Computed hash for {model['filename']}",
"model": model["filename"],
"remaining": len(models_to_lookup),
"total": len(all_models),
})
except Exception as e:
print(f"[ModelSync] Error hashing {model['filename']}: {e}")
return web.json_response({
"status": "error",
"message": f"Failed to hash {model['filename']}: {str(e)}",
"remaining": len(models_to_lookup),
"total": len(all_models),
})
# Step 2: Smart lookup - tries CivitAI, pattern matching, HuggingFace
try:
print(f"[ModelSync] Looking up {model['filename']}...")
info = await self.tracker.smart_lookup(model["filename"], model.get("file_hash"))
if info and info.get("source_url"):
# Found with a URL
self.store.update_model(model["id"], {
"source_url": info.get("source_url"),
"source_type": info.get("source_type", "unknown"),
"remote_hash": info.get("remote_hash"),
})
source_name = info.get("model_name") or info.get("source_hint") or "Unknown"
method = info.get("lookup_method", "unknown")
return web.json_response({
"status": "found",
"message": f"Found {model['filename']}: {source_name} ({method})",
"model": model["filename"],
"source_name": source_name,
"source_url": info.get("source_url"),
"source_type": info.get("source_type"),
"lookup_method": method,
"remaining": len(models_to_lookup) - 1,
"total": len(all_models),
})
elif info and info.get("source_hint"):
# Found a hint but no URL
self.store.update_model(model["id"], {
"source_type": info.get("source_type", "hint"),
"source_url": info.get("source_hint"), # Store hint as URL for reference
})
return web.json_response({
"status": "hint",
"message": f"{model['filename']} likely from {info.get('source_hint')}",
"model": model["filename"],
"source_hint": info.get("source_hint"),
"lookup_method": info.get("lookup_method"),
"remaining": len(models_to_lookup) - 1,
"total": len(all_models),
})
else:
# Not found anywhere
self.store.update_model(model["id"], {
"source_type": "not_found",
})
return web.json_response({
"status": "not_found",
"message": f"{model['filename']} - source unknown",
"model": model["filename"],
"remaining": len(models_to_lookup) - 1,
"total": len(all_models),
})
except Exception as e:
print(f"[ModelSync] Error looking up {model['filename']}: {e}")
return web.json_response({
"status": "error",
"message": f"Lookup failed for {model['filename']}: {str(e)}",
"remaining": len(models_to_lookup),
"total": len(all_models),
})
# Configuration endpoints
async def get_config(self, request: web.Request) -> web.Response: