increase starting lr for sd15

main.py

@@ -213,10 +213,14 @@ def train(
     for i in range(len(text_encoders)):
         grad_norms[f'text_encoder_{i}'] = []
         token_stds[f'text_encoder_{i}'] = {j: [] for j in range(config.n_tokens)}
-
-
+    
     # default value of cold (pre-warmup) optimizer lr:
-    base_lr = 0.5e-5
+    if config.sd_model_version == "sdxl":
+        # let textual_inversion do the work first!
+        base_lr = 0.5e-5
+    elif config.sd_model_version == "sd15":
+        # let lora training kick in soonish
+        base_lr = 1.0e-4
         
     #######################################################################################################
     

trainer/config.py

@@ -74,7 +74,7 @@ class TrainingConfig(BaseModel):
     unet_learning_rate: float = 1.0
     lr_num_cycles: int = 1
     lr_power: float = 1.0
-    sample_imgs_lora_scale: float = 0.7    # Default lora scale for sampling the validation images
+    sample_imgs_lora_scale: float = 0.65    # Default lora scale for sampling the validation images
     dataloader_num_workers: int = 0
     training_attributes: dict = {}
     aspect_ratio_bucketing: bool = False

training_args_face.json

@@ -1,18 +1,18 @@
 {
-    "output_dir": "lora_models/xander_sd15_default_lr",
+    "output_dir": "lora_models/xander_sd15_final",
     "sd_model_version": "sd15",
     "lora_training_urls": "https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/xander_big.zip",
     "concept_mode": "face",
-    "seed": 2,
+    "seed": 0,
     "resolution": 640,
     "train_batch_size": 4,
     "n_sample_imgs": 6,
-    "max_train_steps": 400,
+    "max_train_steps": 600,
     "token_warmup_steps": 0,
     "checkpointing_steps": 100,
     "gradient_accumulation_steps": 1,
 
-    "sample_imgs_lora_scale": 0.7,
+    "sample_imgs_lora_scale": 0.8,
     "n_tokens": 2,
     "ti_lr": 0.001,
     "remove_ti_token_from_prompts": false,
@@ -24,8 +24,8 @@
 
     "unet_lr": 0.001,
     "lora_alpha_multiplier": 1.0,
-    "lora_rank": 16,
-    "use_dora": true,
+    "lora_rank": 12,
+    "use_dora": false,
     "caption_model": "gpt4-v",
     "debug": true
 }

training_args_style_sd15.json

@@ -1,5 +1,5 @@
 {
-    "output_dir": "lora_models/does",
+    "output_dir": "lora_models/does_best",
     "sd_model_version": "sd15",
     "lora_training_urls": "https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/does.zip",
     "concept_mode": "style",
@@ -7,7 +7,7 @@
     "resolution": 640,
     "train_batch_size": 4,
     "n_sample_imgs": 6,
-    "max_train_steps": 700,
+    "max_train_steps": 600,
     "token_warmup_steps": 0,
     "checkpointing_steps": 100,
     "gradient_accumulation_steps": 1,