diff --git a/js/modelsync-panel.js b/js/modelsync-panel.js index 6aa0b1e..f6a08cd 100644 --- a/js/modelsync-panel.js +++ b/js/modelsync-panel.js @@ -177,9 +177,7 @@ class ModelSyncPanel { - - - +
@@ -280,12 +278,6 @@ class ModelSyncPanel { case "scan": await this.scanModels(); break; - case "lookup": - await this.lookupSources(); - break; - case "stop-lookup": - this.stopLookup(); - break; case "check": await this.checkUpdates(); break; @@ -313,99 +305,41 @@ class ModelSyncPanel { } } - async lookupSources() { - this.lookupRunning = true; - let foundCount = 0; - let notFoundCount = 0; - let processedCount = 0; - - // Show stop button, hide lookup button - const lookupBtn = this.element?.querySelector('[data-action="lookup"]'); - const stopBtn = this.element?.querySelector('[data-action="stop-lookup"]'); - if (lookupBtn) lookupBtn.style.display = 'none'; - if (stopBtn) stopBtn.style.display = 'inline-block'; - - const processNext = async () => { - if (!this.lookupRunning) { - if (lookupBtn) lookupBtn.style.display = 'inline-block'; - if (stopBtn) stopBtn.style.display = 'none'; - this.setStatus(`Lookup stopped. Found ${foundCount}, not found ${notFoundCount}.`); - await this.refreshData(); - return; - } - - try { - const res = await fetch("/modelsync/lookup-sources", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({}), - }); - const data = await res.json(); - - if (data.status === "complete") { - this.lookupRunning = false; - if (lookupBtn) lookupBtn.style.display = 'inline-block'; - if (stopBtn) stopBtn.style.display = 'none'; - this.setStatus(`Lookup complete. Found ${foundCount} on CivitAI, ${notFoundCount} not found.`); - await this.refreshData(); - return; - } - - if (data.status === "found") { - foundCount++; - processedCount++; - } else if (data.status === "hint") { - foundCount++; // Count hints as partial success - processedCount++; - } else if (data.status === "not_found") { - notFoundCount++; - processedCount++; - } else if (data.status === "hashing") { - // Just hashed, will lookup next call - } - - const progress = data.total > 0 ? Math.round(((data.total - data.remaining) / data.total) * 100) : 0; - this.setStatus(`[${progress}%] ${data.message} (${foundCount} found, ${data.remaining} remaining)`); - - // Update UI periodically - if (processedCount % 5 === 0) { - await this.refreshData(); - } - - // Continue to next model - setTimeout(processNext, 100); - - } catch (error) { - this.lookupRunning = false; - if (lookupBtn) lookupBtn.style.display = 'inline-block'; - if (stopBtn) stopBtn.style.display = 'none'; - this.setStatus("Lookup error: " + error.message); - await this.refreshData(); - } - }; - - this.setStatus("Starting source lookup..."); - processNext(); - } - - stopLookup() { - this.lookupRunning = false; - this.setStatus("Stopping lookup..."); - } - async checkUpdates() { - this.setStatus("Checking for updates..."); + this.setStatus("Checking updates (hashing, finding sources, checking versions)..."); + + // Disable button during check + const checkBtn = this.element?.querySelector('[data-action="check"]'); + if (checkBtn) { + checkBtn.disabled = true; + checkBtn.textContent = "Checking..."; + } + try { const res = await fetch("/modelsync/check", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({}), + body: JSON.stringify({ concurrent: 5 }), }); const data = await res.json(); - this.setStatus(`Checked ${data.checked} models. Updates available: ${data.updates_available}`); + + const parts = []; + if (data.hashed > 0) parts.push(`${data.hashed} hashed`); + if (data.sources_found > 0) parts.push(`${data.sources_found} sources found`); + if (data.checked > 0) parts.push(`${data.checked} checked`); + if (data.updates_available > 0) parts.push(`${data.updates_available} updates!`); + if (data.errors > 0) parts.push(`${data.errors} errors`); + + const summary = parts.length > 0 ? parts.join(", ") : "No models to process"; + this.setStatus(`Done: ${summary}`); await this.refreshData(); } catch (error) { this.setStatus("Check failed: " + error.message); + } finally { + if (checkBtn) { + checkBtn.disabled = false; + checkBtn.textContent = "Check Updates"; + } } } diff --git a/server.py b/server.py index 1f30d7a..43ca98a 100644 --- a/server.py +++ b/server.py @@ -71,7 +71,6 @@ class ModelSyncAPI: app.router.add_post("/modelsync/check", self.check_updates) app.router.add_post("/modelsync/download/{id}", self.download_update) - app.router.add_post("/modelsync/lookup-sources", self.lookup_sources) app.router.add_get("/modelsync/config", self.get_config) app.router.add_post("/modelsync/config", self.set_config) @@ -208,50 +207,115 @@ class ModelSyncAPI: # Version check endpoints async def check_updates(self, request: web.Request) -> web.Response: - """Check models for updates.""" + """ + Unified check: finds sources and checks for updates concurrently. + Processes models in batches for efficiency. + """ + import asyncio + data = await request.json() if request.body_exists else {} model_ids = data.get("model_ids") + concurrent_limit = data.get("concurrent", 5) + all_models = self.store.get_all_models() if model_ids: - models = [self.store.get_model(mid) for mid in model_ids] - models = [m for m in models if m and m.get("source_url")] - else: - models = self.store.get_all_models() - models = [m for m in models if m.get("source_url")] + all_models = [m for m in all_models if m["id"] in model_ids] - if not models: - return web.json_response({"checked": 0, "updates_available": 0}) + if not all_models: + return web.json_response({ + "total": 0, "checked": 0, "sources_found": 0, + "updates_available": 0, "status": "complete" + }) - results = await self.tracker.batch_check(models) - updates_found = 0 + # Track already-checked source types + checked_types = {"not_found", "civitai", "huggingface", "github", "hint"} - for model in models: - model_id = model["id"] - if model_id in results: - remote_info = results[model_id] + stats = { + "total": len(all_models), + "hashed": 0, + "sources_found": 0, + "checked": 0, + "updates_available": 0, + "errors": 0, + } - is_update = self.tracker.is_update_available( - local_hash=model.get("file_hash"), - local_size=model.get("file_size"), - remote_hash=remote_info.get("hash"), - remote_size=remote_info.get("size"), - ) + semaphore = asyncio.Semaphore(concurrent_limit) - self.store.update_model(model_id, { - "remote_hash": remote_info.get("hash"), - "remote_size": remote_info.get("size"), - "remote_modified": remote_info.get("modified"), - "last_checked": datetime.now().isoformat(), - "update_available": 1 if is_update else 0, - }) + async def process_model(model): + """Process a single model: hash → find source → check update.""" + async with semaphore: + model_id = model["id"] + filename = model["filename"] - if is_update: - updates_found += 1 + try: + # Step 1: Compute hash if missing + if not model.get("file_hash"): + try: + file_hash = compute_sha256_chunked(model["path"]) + self.store.update_model(model_id, {"file_hash": file_hash}) + model["file_hash"] = file_hash + stats["hashed"] += 1 + except Exception as e: + print(f"[ModelSync] Hash error {filename}: {e}") + stats["errors"] += 1 + return - return web.json_response({ - "checked": len(models), - "updates_available": updates_found, - }) + # Step 2: Find source if missing + if not model.get("source_url") and model.get("source_type") not in checked_types: + info = await self.tracker.smart_lookup(filename, model.get("file_hash")) + + if info and info.get("source_url"): + self.store.update_model(model_id, { + "source_url": info.get("source_url"), + "source_type": info.get("source_type", "unknown"), + }) + model["source_url"] = info.get("source_url") + model["source_type"] = info.get("source_type") + stats["sources_found"] += 1 + elif info and info.get("source_hint"): + self.store.update_model(model_id, { + "source_type": info.get("source_type", "hint"), + "source_url": info.get("source_hint"), + }) + model["source_url"] = info.get("source_hint") + stats["sources_found"] += 1 + else: + self.store.update_model(model_id, {"source_type": "not_found"}) + + # Step 3: Check for updates if we have a source URL + source_url = model.get("source_url") + if source_url and source_url.startswith("http"): + remote_info = await self.tracker.check_version(source_url) + + if remote_info: + is_update = self.tracker.is_update_available( + local_hash=model.get("file_hash"), + local_size=model.get("file_size"), + remote_hash=remote_info.get("hash"), + remote_size=remote_info.get("size"), + ) + + self.store.update_model(model_id, { + "remote_hash": remote_info.get("hash"), + "remote_size": remote_info.get("size"), + "remote_modified": remote_info.get("modified"), + "last_checked": datetime.now().isoformat(), + "update_available": 1 if is_update else 0, + }) + + stats["checked"] += 1 + if is_update: + stats["updates_available"] += 1 + + except Exception as e: + print(f"[ModelSync] Error processing {filename}: {e}") + stats["errors"] += 1 + + # Process all models concurrently + await asyncio.gather(*[process_model(m) for m in all_models]) + + stats["status"] = "complete" + return web.json_response(stats) async def download_update(self, request: web.Request) -> web.Response: """Download update for a specific model.""" @@ -271,126 +335,6 @@ class ModelSyncAPI: "model_id": model_id, }) - async def lookup_sources(self, request: web.Request) -> web.Response: - """ - Look up source URL for ONE model at a time by querying CivitAI with file hash. - Call repeatedly to process all models with progress feedback. - """ - data = await request.json() if request.body_exists else {} - - # Get all models that haven't been looked up yet - # Exclude: models with source_url, or source_type indicating already checked - all_models = self.store.get_all_models() - checked_types = {"not_found", "civitai", "huggingface", "github", "hint"} - models_to_lookup = [ - m for m in all_models - if not m.get("source_url") and m.get("source_type") not in checked_types - ] - - if not models_to_lookup: - return web.json_response({ - "status": "complete", - "message": "All models already have sources", - "remaining": 0, - "total": len(all_models), - }) - - # Find next model to process (prioritize ones with hashes) - model = None - for m in models_to_lookup: - if m.get("file_hash"): - model = m - break - - # If no model has hash, pick first one to hash - if not model: - model = models_to_lookup[0] - - # Step 1: Compute hash if needed - if not model.get("file_hash"): - try: - print(f"[ModelSync] Hashing {model['filename']}...") - file_hash = compute_sha256_chunked(model["path"]) - self.store.update_model(model["id"], {"file_hash": file_hash}) - model["file_hash"] = file_hash - return web.json_response({ - "status": "hashing", - "message": f"Computed hash for {model['filename']}", - "model": model["filename"], - "remaining": len(models_to_lookup), - "total": len(all_models), - }) - except Exception as e: - print(f"[ModelSync] Error hashing {model['filename']}: {e}") - return web.json_response({ - "status": "error", - "message": f"Failed to hash {model['filename']}: {str(e)}", - "remaining": len(models_to_lookup), - "total": len(all_models), - }) - - # Step 2: Smart lookup - tries CivitAI, pattern matching, HuggingFace - try: - print(f"[ModelSync] Looking up {model['filename']}...") - info = await self.tracker.smart_lookup(model["filename"], model.get("file_hash")) - - if info and info.get("source_url"): - # Found with a URL - self.store.update_model(model["id"], { - "source_url": info.get("source_url"), - "source_type": info.get("source_type", "unknown"), - "remote_hash": info.get("remote_hash"), - }) - source_name = info.get("model_name") or info.get("source_hint") or "Unknown" - method = info.get("lookup_method", "unknown") - return web.json_response({ - "status": "found", - "message": f"Found {model['filename']}: {source_name} ({method})", - "model": model["filename"], - "source_name": source_name, - "source_url": info.get("source_url"), - "source_type": info.get("source_type"), - "lookup_method": method, - "remaining": len(models_to_lookup) - 1, - "total": len(all_models), - }) - elif info and info.get("source_hint"): - # Found a hint but no URL - self.store.update_model(model["id"], { - "source_type": info.get("source_type", "hint"), - "source_url": info.get("source_hint"), # Store hint as URL for reference - }) - return web.json_response({ - "status": "hint", - "message": f"{model['filename']} likely from {info.get('source_hint')}", - "model": model["filename"], - "source_hint": info.get("source_hint"), - "lookup_method": info.get("lookup_method"), - "remaining": len(models_to_lookup) - 1, - "total": len(all_models), - }) - else: - # Not found anywhere - self.store.update_model(model["id"], { - "source_type": "not_found", - }) - return web.json_response({ - "status": "not_found", - "message": f"{model['filename']} - source unknown", - "model": model["filename"], - "remaining": len(models_to_lookup) - 1, - "total": len(all_models), - }) - - except Exception as e: - print(f"[ModelSync] Error looking up {model['filename']}: {e}") - return web.json_response({ - "status": "error", - "message": f"Lookup failed for {model['filename']}: {str(e)}", - "remaining": len(models_to_lookup), - "total": len(all_models), - }) - # Configuration endpoints async def get_config(self, request: web.Request) -> web.Response: