feat: add Storage Optimizer with SHA-256 hashing and Civitai hash lookup

This commit is contained in:
Thomas Kippster
2026-04-26 17:48:33 +02:00
parent 3c99c5358d
commit e0d68c8cfb
9 changed files with 477 additions and 38 deletions
@@ -0,0 +1,119 @@
package de.tki.comfymodels.service;
import de.tki.comfymodels.service.impl.ConfigService;
import de.tki.comfymodels.service.impl.ModelHashRegistry;
import de.tki.comfymodels.service.impl.ModelValidator;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
public class ModelHashRegistryTest {
@TempDir
Path tempDir;
private ConfigService configService;
private IModelValidator validator;
private ModelHashRegistry registry;
private String lastCalculatedHash = "hash123";
private int calculationCount = 0;
@BeforeEach
public void setup() throws Exception {
configService = new ConfigService(null) {
@Override
public File getFileInAppData(String filename) {
return tempDir.resolve(filename).toFile();
}
};
validator = new IModelValidator() {
@Override
public ValidationResult validateFile(File file) { return new ValidationResult(true, "OK", file.getAbsolutePath()); }
@Override
public String calculateHash(File file) {
calculationCount++;
return lastCalculatedHash;
}
};
registry = new ModelHashRegistry();
// Use reflection to set private fields since we aren't using @InjectMocks
setField(registry, "configService", configService);
setField(registry, "validator", validator);
registry.load();
}
private void setField(Object obj, String fieldName, Object value) throws Exception {
Field field = obj.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(obj, value);
}
@Test
public void testGetOrCalculateHash() throws Exception {
Path file = tempDir.resolve("model.safetensors");
Files.writeString(file, "content");
File modelFile = file.toFile();
lastCalculatedHash = "hash123";
calculationCount = 0;
// First call: should calculate
String hash1 = registry.getOrCalculateHash(modelFile);
assertEquals("hash123", hash1);
assertEquals(1, calculationCount);
// Second call: should use cache because size and timestamp are same
String hash2 = registry.getOrCalculateHash(modelFile);
assertEquals("hash123", hash2);
assertEquals(1, calculationCount);
// Change content -> size stays same, but timestamp changes (usually)
// To be sure for the test, we wait a bit or just assume size change
Files.writeString(file, "content has changed now");
File updatedFile = file.toFile();
lastCalculatedHash = "new_hash";
String hash3 = registry.getOrCalculateHash(updatedFile);
assertEquals("new_hash", hash3);
assertEquals(2, calculationCount); // Re-calculation triggered!
}
@Test
public void testFindPathByHash() throws Exception {
Path file = tempDir.resolve("duplicate.safetensors");
Files.writeString(file, "duplicate content");
File modelFile = file.toFile();
lastCalculatedHash = "duplicate_hash";
registry.getOrCalculateHash(modelFile);
Optional<String> foundPath = registry.findPathByHash("duplicate_hash");
assertTrue(foundPath.isPresent());
assertEquals(modelFile.getAbsolutePath(), foundPath.get());
}
@Test
public void testUnregister() throws Exception {
Path file = tempDir.resolve("to_be_deleted.safetensors");
Files.writeString(file, "soon gone");
File modelFile = file.toFile();
lastCalculatedHash = "temp_hash";
registry.getOrCalculateHash(modelFile);
assertTrue(registry.findPathByHash("temp_hash").isPresent());
registry.unregister(modelFile);
assertFalse(registry.findPathByHash("temp_hash").isPresent());
}
}
@@ -0,0 +1,47 @@
package de.tki.comfymodels.service;
import de.tki.comfymodels.service.impl.ModelValidator;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import static org.junit.jupiter.api.Assertions.*;
public class ModelHashTest {
@TempDir
Path tempDir;
@Test
public void testCalculateSHA256() throws Exception {
ModelValidator validator = new ModelValidator();
Path testFile = tempDir.resolve("test_model.bin");
byte[] data = "ComfyUI Model Data for Hashing Test".getBytes(java.nio.charset.StandardCharsets.UTF_8);
Files.write(testFile, data);
String hash = validator.calculateHash(testFile.toFile());
assertNotNull(hash);
assertEquals(64, hash.length());
// echo -n "ComfyUI Model Data for Hashing Test" | sha256sum
String expectedHash = "5022ad6b25eac3358b18aa8ef8ee79d06833c43bed545bed1a5d45057d3054e2";
assertEquals(expectedHash, hash);
}
@Test
public void testEmptyFileHash() throws Exception {
ModelValidator validator = new ModelValidator();
Path emptyFile = tempDir.resolve("empty.bin");
Files.createFile(emptyFile);
String hash = validator.calculateHash(emptyFile.toFile());
// SHA-256 for empty string
String expectedEmptyHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
assertEquals(expectedEmptyHash, hash);
}
}