nodes/thomaskippster-comfymodeldownloader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add markdown link extraction and HF URL normalization in ComfyModelAnalyzer and ModelSearchService

Browse Source
This commit is contained in:
Thomas Kippster
2026-04-22 19:39:11 +02:00
parent 6116eb6d63
commit 60320e13a2
24 changed files with 1661 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/de/tki/comfymodels/service/impl/DefaultDownloadManager.java
+197 -54
View File
@@ -22,28 +22,78 @@ public class DefaultDownloadManager implements IDownloadManager {
private final HttpClient httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build();
private volatile boolean isPaused = false;
private volatile boolean isStopped = false;
private volatile boolean[] currentSelection;
private final java.util.Set<Integer> completedIndices = java.util.concurrent.ConcurrentHashMap.newKeySet();
@Autowired
private ConfigService configService;
private void safeUpdateStatus(int index, String status, BiConsumer<Integer, String> statusUpdater) {
if (completedIndices.contains(index)) return;
if (status.contains("✅ Finished") || status.contains("Stopped") || status.contains("Skipped") || status.contains("Already exists")) {
completedIndices.add(index);
}
statusUpdater.accept(index, status);
}
@Override
public void startQueue(List<ModelInfo> models, String baseDir, BiConsumer<Integer, String> statusUpdater, Runnable onFinished) {
public void startQueue(List<ModelInfo> models, boolean[] selectedIndices, String baseDir, BiConsumer<Integer, String> statusUpdater, Runnable onFinished) {
isStopped = false;
isPaused = false;
this.currentSelection = selectedIndices;
completedIndices.clear();
new Thread(() -> {
try {
java.util.concurrent.CompletableFuture<?>[] futures = new java.util.concurrent.CompletableFuture[models.size()];
for (int i = 0; i < models.size(); i++) {
if (isStopped) {
for (int j = i; j < models.size(); j++) {
futures[j] = java.util.concurrent.CompletableFuture.completedFuture(null);
}
break;
}
while (isPaused && !isStopped) {
try {
Thread.sleep(200);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
final int index = i;
if (!isSelected(index)) {
safeUpdateStatus(index, "Skipped (Not Selected)", statusUpdater);
futures[i] = java.util.concurrent.CompletableFuture.completedFuture(null);
continue;
}
ModelInfo info = models.get(i);
futures[i] = java.util.concurrent.CompletableFuture.runAsync(() -> {
if (isStopped) { statusUpdater.accept(index, "Stopped"); return; }
if (info.getUrl() == null || info.getUrl().startsWith("MISSING")) { statusUpdater.accept(index, "Skipped (No URL)"); return; }
if (isStopped) { safeUpdateStatus(index, "Stopped", statusUpdater); return; }
if (!isSelected(index)) { safeUpdateStatus(index, "Skipped (Not Selected)", statusUpdater); return; }
if (info.getUrl() == null || info.getUrl().startsWith("MISSING")) { safeUpdateStatus(index, "Skipped (No URL)", statusUpdater); return; }
try {
Path targetDir = Paths.get(baseDir, info.getType());
String subPath = info.getSave_path();
if (subPath == null || subPath.isEmpty() || "default".equalsIgnoreCase(subPath)) {
subPath = info.getType() != null ? info.getType() : "checkpoints";
}
String finalBaseDir = baseDir != null ? baseDir : ".";
Path targetDir = Paths.get(finalBaseDir, subPath);
Files.createDirectories(targetDir);
downloadWithResume(info, targetDir.resolve(info.getName()), index, statusUpdater);
} catch (Exception e) { statusUpdater.accept(index, "Error: " + e.getMessage()); }
downloadWithResume(info, targetDir.resolve(info.getName()), index, statusUpdater);
} catch (Exception e) {
if (isStopped || !isSelected(index)) {
safeUpdateStatus(index, !isSelected(index) ? "Skipped (Unchecked)" : "Stopped", statusUpdater);
} else {
String msg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
safeUpdateStatus(index, "Error: " + msg, statusUpdater);
}
}
}, executor);
}
java.util.concurrent.CompletableFuture.allOf(futures).join();
@@ -51,62 +101,155 @@ public class DefaultDownloadManager implements IDownloadManager {
}).start();
}
private void downloadWithResume(ModelInfo info, Path targetFile, int index, BiConsumer<Integer, String> statusUpdater) throws Exception {
File file = targetFile.toFile();
// ANTI-STUB LOGIK: Wenn Datei < 10KB, löschen wir sie und fangen neu an (da es wahrscheinlich ein LFS-Pointer ist)
if (file.exists() && file.length() < 10240) {
file.delete();
}
long existingFileSize = file.exists() ? file.length() : 0;
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(URI.create(info.getUrl())).header("User-Agent", "Mozilla/5.0");
String hfToken = configService.getHfToken();
if (info.getUrl().contains("huggingface.co") && !hfToken.isEmpty()) {
builder.header("Authorization", "Bearer " + hfToken);
}
private boolean isSelected(int index) {
return currentSelection == null || (index < currentSelection.length && currentSelection[index]);
}
if (existingFileSize > 0) builder.header("Range", "bytes=" + existingFileSize + "-");
@Override
public void updateSelection(boolean[] selectedIndices) {
this.currentSelection = selectedIndices;
}
HttpResponse<InputStream> response = httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofInputStream());
int statusCode = response.statusCode();
if (statusCode == 401 || statusCode == 403) {
statusUpdater.accept(index, "❌ Auth Required (Token?)");
return;
}
private void downloadWithResume(ModelInfo info, Path targetFile, int index, BiConsumer<Integer, String> statusUpdater) {
try {
if (waitForPauseAndCheckSelection(index, statusUpdater)) return;
long contentLen = response.headers().firstValueAsLong("Content-Length").orElse(0L);
long totalBytes = (statusCode == 206) ? contentLen + existingFileSize : contentLen;
File file = targetFile.toFile();
String hfToken = configService.getHfToken();
// VERIFIZIERUNG: Falls die Gesamtgröße extrem klein ist, ist es ein LFS Stub
if (totalBytes > 0 && totalBytes < 5000 && info.getName().endsWith(".safetensors")) {
statusUpdater.accept(index, "❌ LFS Stub detected (Token?)");
return;
}
if (file.exists() && file.length() < 10240) {
file.delete();
}
if (statusCode == 200) existingFileSize = 0;
long existingFileSize = file.exists() ? file.length() : 0;
try (InputStream is = response.body(); RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.seek(existingFileSize);
byte[] buffer = new byte[65536];
long downloaded = existingFileSize;
int read;
long lastUpdate = 0;
while ((read = is.read(buffer)) != -1) {
if (isStopped) return;
while (isPaused) { statusUpdater.accept(index, "Paused..."); Thread.sleep(1000); }
raf.write(buffer, 0, read);
downloaded += read;
long now = System.currentTimeMillis();
if (now - lastUpdate > 800) {
statusUpdater.accept(index, "Downloading: " + (totalBytes > 0 ? (downloaded * 100 / totalBytes) : "?") + "% (" + formatSize(downloaded) + ")");
lastUpdate = now;
if (waitForPauseAndCheckSelection(index, statusUpdater)) return;
HttpRequest.Builder headBuilder = HttpRequest.newBuilder()
.uri(URI.create(info.getUrl()))
.header("User-Agent", "Mozilla/5.0")
.method("HEAD", HttpRequest.BodyPublishers.noBody());
if (info.getUrl().contains("huggingface.co") && hfToken != null && !hfToken.isEmpty()) {
headBuilder.header("Authorization", "Bearer " + hfToken);
}
HttpResponse<Void> headResponse = httpClient.send(headBuilder.build(), HttpResponse.BodyHandlers.discarding());
if (headResponse.statusCode() == 401 || headResponse.statusCode() == 403) {
safeUpdateStatus(index, "❌ Auth Required (Token?)", statusUpdater);
return;
}
long totalRemoteSize = headResponse.headers().firstValueAsLong("Content-Length").orElse(0L);
if (existingFileSize > 0 && totalRemoteSize > 0 && existingFileSize == totalRemoteSize) {
safeUpdateStatus(index, "✅ Already exists", statusUpdater);
return;
}
if (waitForPauseAndCheckSelection(index, statusUpdater)) return;
HttpRequest.Builder downloadBuilder = HttpRequest.newBuilder()
.uri(URI.create(info.getUrl()))
.header("User-Agent", "Mozilla/5.0");
if (info.getUrl().contains("huggingface.co") && hfToken != null && !hfToken.isEmpty()) {
downloadBuilder.header("Authorization", "Bearer " + hfToken);
}
if (existingFileSize > 0) downloadBuilder.header("Range", "bytes=" + existingFileSize + "-");
HttpResponse<InputStream> response = httpClient.send(downloadBuilder.build(), HttpResponse.BodyHandlers.ofInputStream());
int statusCode = response.statusCode();
if (statusCode == 416) {
safeUpdateStatus(index, "✅ Already exists", statusUpdater);
return;
}
if (statusCode == 401 || statusCode == 403) {
safeUpdateStatus(index, "❌ Auth Required (Token?)", statusUpdater);
return;
}
long contentLen = response.headers().firstValueAsLong("Content-Length").orElse(0L);
long totalBytes = (statusCode == 206) ? contentLen + existingFileSize : contentLen;
if (totalBytes > 0 && totalBytes < 5000 && info.getName().endsWith(".safetensors")) {
safeUpdateStatus(index, "❌ LFS Stub detected (Token?)", statusUpdater);
return;
}
if (statusCode == 200) existingFileSize = 0;
try (InputStream is = response.body(); RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
raf.seek(existingFileSize);
byte[] buffer = new byte[65536];
long downloaded = existingFileSize;
int read;
long lastUpdate = 0;
while ((read = is.read(buffer)) != -1) {
if (isStopped || !isSelected(index) || Thread.currentThread().isInterrupted()) {
safeUpdateStatus(index, !isSelected(index) ? "Skipped (Unchecked)" : "Stopped", statusUpdater);
return;
}
if (isPaused) {
if (waitForPauseAndCheckSelection(index, statusUpdater)) return;
safeUpdateStatus(index, "Resuming...", statusUpdater);
}
raf.write(buffer, 0, read);
downloaded += read;
long now = System.currentTimeMillis();
if (now - lastUpdate > 800) {
safeUpdateStatus(index, "Downloading: " + (totalBytes > 0 ? (downloaded * 100 / totalBytes) : "?") + "% (" + formatSize(downloaded) + ")", statusUpdater);
lastUpdate = now;
}
}
}
if (isStopped || !isSelected(index)) {
safeUpdateStatus(index, !isSelected(index) ? "Skipped (Unchecked)" : "Stopped", statusUpdater);
} else {
if (totalBytes > 0 && file.length() < totalBytes) {
safeUpdateStatus(index, "❌ Incomplete (" + formatSize(file.length()) + "/" + formatSize(totalBytes) + ")", statusUpdater);
} else {
safeUpdateStatus(index, "✅ Finished", statusUpdater);
}
}
} catch (Exception e) {
// Falls es ein Timing-Problem ist (Flagge wird gesetzt während Exception fliegt)
try { Thread.sleep(50); } catch (InterruptedException ignored) {}
if (isStopped || !isSelected(index) || Thread.currentThread().isInterrupted()) {
safeUpdateStatus(index, !isSelected(index) ? "Skipped (Unchecked)" : "Stopped", statusUpdater);
} else {
String msg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName();
safeUpdateStatus(index, "Error: " + msg, statusUpdater);
}
}
statusUpdater.accept(index, "Finished");
}
private boolean waitForPauseAndCheckSelection(int index, BiConsumer<Integer, String> statusUpdater) {
try {
if (isPaused) {
safeUpdateStatus(index, "Paused", statusUpdater);
}
while (isPaused && !isStopped && isSelected(index)) {
Thread.sleep(200);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
if (isStopped || !isSelected(index)) {
safeUpdateStatus(index, !isSelected(index) ? "Skipped (Unchecked)" : "Stopped", statusUpdater);
return true;
}
return false;
}
private String formatSize(long bytes) {
@@ -118,4 +261,4 @@ public class DefaultDownloadManager implements IDownloadManager {
@Override public void togglePause() { isPaused = !isPaused; }
@Override public void stop() { isStopped = true; isPaused = false; }
@Override public boolean isPaused() { return isPaused; }
}
}