update todos

README.md

@@ -41,6 +41,7 @@ Algo:
 - test if textual inversion training can also happen with prodigy_optimizer
 - add CLIP_similarity token warmup (txt = Done, img = TODO) or (aesthetic gradients: https://github.com/vicgalle/stable-diffusion-aesthetic-gradients/tree/main)
 - improve data augmentation, eg by adding outpainted, smaller versions of faces / objects
+- figure out why the initial onset of learning in the LoRa / Dora causes a temporary drop in img quality
 - currently some sub-optimal stuff can happen in preprocess() when there's less than 3 or more than 45 imgs, try to improve this
 - make a separate gpt4-v call to query the concept-description using a random sample of the training imgs, assembled into a single img grid
 

main.py

@@ -303,7 +303,7 @@ def train(
             captions = list(captions)
                     
             prompt_embeds, pooled_prompt_embeds, add_time_ids = get_conditioning_signals(
-                config, pipe, captions, text_encoders
+                config, pipe, captions
             )
             
             # Sample noise that we'll add to the latents:

trainer/inference.py

@@ -99,7 +99,7 @@ def prepare_prompt_for_lora(prompt, lora_path, interpolation=False, verbose=True
 
 
 
-def get_conditioning_signals(config, pipe, captions, text_encoders):
+def get_conditioning_signals(config, pipe, captions):
     conditioning_signals = pipe.encode_prompt(
         prompt = captions,
         device = pipe.unet.device,

trainer/loss.py

@@ -129,7 +129,7 @@ class ConditioningRegularizer:
     def _compute_tok_regularization_loss(self, pipe):
         reg_captions = [caption.replace("TOK", self.token_replacement) for caption in self.reg_captions]
         reg_prompt_embeds, reg_pooled_prompt_embeds, reg_add_time_ids = get_conditioning_signals(
-            self.config, pipe, reg_captions, self.embedding_handler.text_encoders
+            self.config, pipe, reg_captions
         )
 
         reg_conditioning_norms = reg_prompt_embeds.norm(dim=-1).mean(dim=0)

training_args_test.json

@@ -1,6 +1,6 @@
 {
     "output_dir": "lora_models/banny_test",
-    "sd_model_version": "sd15",
+    "sd_model_version": "sdxl",
     "lora_training_urls": "/home/xander/Downloads/datasets/banny_best_uncaptioned",
     "concept_mode": "object",
     "seed": 1,
@@ -9,14 +9,14 @@
     "train_batch_size": 4,
     "n_sample_imgs": 6,
     "max_train_steps": 360,
-    "token_warmup_steps": 40,
+    "token_warmup_steps": 100,
     "checkpointing_steps": 60,
     "gradient_accumulation_steps": 1,
     "n_tokens": 3,
     "ti_lr": 0.001,
     "ti_weight_decay": 0.0002,
     "off_ratio_power": 0.05,
-    "prodigy_d_coef": 0.5,
+    "prodigy_d_coef": 0.1,
     "lora_weight_decay": 0.0001,
     "l1_penalty": 0.1,
     "noise_offset": 0.05,