diff --git a/.gitignore b/.gitignore index fedbdcd..229ed27 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,24 @@ -models -lora_models* -datasets/ -remove + cache __pycache__ -scripts/gridsearch* + +models +lora_models* +datasets + *.tar .env .cog -xander*.sh .huggingface -tests/ train.py -debug/* rendered_images* -!debug/*.py + gridsearch* -xander* aesthetic_score_best_model.pth + +# experiment folders: +conditioning_spaces/ +xander*.sh training_args_x_*.json -eval_images/ -test.json \ No newline at end of file +xander* +debug/* diff --git a/scripts/evaluate_gridsearch.py b/scripts/evaluate_gridsearch.py index 6d16f9d..44948f4 100644 --- a/scripts/evaluate_gridsearch.py +++ b/scripts/evaluate_gridsearch.py @@ -11,10 +11,14 @@ from sklearn.metrics import r2_score # Define paths -render_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/lora_models/unet2" -config_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/gridsearch_configs/unet_adam_object" +render_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/lora_models/XANDER_SD15_SWEEP/400" +config_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/gridsearch_configs/sd15_face_sweep" -ignore_threshold_relative = 0.25 # ignore any datapoint with a score below this threshold +ignore_threshold_relative = 0.0 # ignore any datapoint with a score below this threshold + +filters = { + "resolution": 512 +} output_dir = f"gridsearch_configs/results/{os.path.basename(config_dir)}" output_suffix = f"{os.path.basename(render_dir)}" @@ -40,6 +44,11 @@ for i, exp_subdir in enumerate(sorted(os.listdir(render_dir))): if os.path.isfile(json_path): with open(json_path, 'r') as file: config = json.load(file) + + # Filter out experiments that do not match the filters + if not all(config[key] == value for key, value in filters.items()): + continue + # Step 4: Append all key/value pairs to the total experiment dictionary for key, value in config.items(): parameters[key]['values'].append(value) @@ -47,6 +56,13 @@ for i, exp_subdir in enumerate(sorted(os.listdir(render_dir))): else: print(f"Could not find JSON file for experiment {exp_subdir}") + +# Print the parameters['output_dir'] with the highest scores (there are usually multiple ties): +max_score = max(parameters['output_dir']['scores']) +best_output_dirs = [output_dir for output_dir, score in zip(parameters['output_dir']['values'], parameters['output_dir']['scores']) if score == max_score] +for best_output_dir in best_output_dirs: + print(f"Best output_dir: {best_output_dir} with score {max_score}") + import numpy as np import matplotlib.pyplot as plt import seaborn as sns diff --git a/scripts/test_inference.py b/scripts/test_inference.py index 71ff10c..6ae6db3 100644 --- a/scripts/test_inference.py +++ b/scripts/test_inference.py @@ -17,11 +17,11 @@ from trainer.checkpoint import load_checkpoint if __name__ == "__main__": - pretrained_model = pretrained_models['sdxl'] - lora_path = 'lora_models/xander_one_img--23_15-25-00-sdxl_face_lora_512_1.0_gpt4-v/checkpoints/checkpoint-360' + model_version = "sd15" + lora_path = 'lora_models/XANDER_SD15_SWEEP/sd15_face_sweep__004--29_20-43-17-sd15_face_dora_640_1.0_blip_800/checkpoints/checkpoint-800' lora_scales = np.linspace(0.6, 0.9, 4) token_scale = None # None means it well get automatically set using lora_scale - render_size = (1024, 1024) # H,W + render_size = (576, 704) # H,W n_imgs = 14 n_loops = 2 @@ -32,6 +32,7 @@ if __name__ == "__main__": ##################################################################################### + pretrained_model = pretrained_models[model_version] output_dir = f'rendered_images/{lora_path.split("/")[-1]}' os.makedirs(output_dir, exist_ok=True) @@ -39,8 +40,8 @@ if __name__ == "__main__": pick_best_gpu_id() pipe = load_checkpoint( - pretrained_model_version="sdxl", - pretrained_model_path=pretrained_models["sdxl"]["path"], + pretrained_model_version=model_version, + pretrained_model_path=pretrained_model["path"], checkpoint_folder=lora_path, is_lora=True, device="cuda:0" diff --git a/sd15_face_sweep.sh b/sd15_face_sweep.sh new file mode 100644 index 0000000..b177d17 --- /dev/null +++ b/sd15_face_sweep.sh @@ -0,0 +1,202 @@ +#!/bin/bash + +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_000.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_001.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_002.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_003.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_004.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_005.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_006.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_007.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_008.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_009.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_010.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_011.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_012.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_013.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_014.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_015.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_016.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_017.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_018.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_019.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_020.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_021.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_022.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_023.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_024.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_025.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_026.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_027.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_028.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_029.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_030.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_031.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_032.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_033.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_034.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_035.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_036.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_037.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_038.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_039.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_040.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_041.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_042.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_043.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_044.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_045.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_046.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_047.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_048.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_049.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_050.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_051.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_052.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_053.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_054.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_055.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_056.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_057.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_058.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_059.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_060.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_061.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_062.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_063.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_064.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_065.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_066.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_067.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_068.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_069.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_070.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_071.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_072.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_073.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_074.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_075.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_076.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_077.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_078.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_079.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_080.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_081.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_082.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_083.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_084.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_085.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_086.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_087.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_088.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_089.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_090.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_091.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_092.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_093.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_094.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_095.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_096.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_097.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_098.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_099.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_100.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_101.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_102.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_103.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_104.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_105.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_106.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_107.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_108.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_109.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_110.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_111.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_112.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_113.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_114.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_115.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_116.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_117.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_118.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_119.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_120.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_121.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_122.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_123.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_124.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_125.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_126.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_127.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_128.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_129.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_130.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_131.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_132.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_133.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_134.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_135.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_136.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_137.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_138.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_139.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_140.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_141.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_142.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_143.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_144.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_145.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_146.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_147.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_148.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_149.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_150.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_151.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_152.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_153.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_154.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_155.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_156.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_157.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_158.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_159.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_160.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_161.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_162.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_163.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_164.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_165.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_166.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_167.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_168.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_169.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_170.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_171.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_172.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_173.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_174.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_175.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_176.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_177.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_178.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_179.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_180.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_181.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_182.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_183.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_184.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_185.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_186.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_187.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_188.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_189.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_190.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_191.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_192.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_193.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_194.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_195.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_196.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_197.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_198.json +python main.py gridsearch_configs/sd15_face_sweep/sd15_face_sweep_199.json diff --git a/trainer/config.py b/trainer/config.py index c99bc56..1732acf 100644 --- a/trainer/config.py +++ b/trainer/config.py @@ -27,7 +27,7 @@ class TrainingConfig(BaseModel): unet_optimizer_type: Literal["adamw", "prodigy"] = "adamw" unet_lr_warmup_steps: int = None # slowly increase the learning rate of the adamw unet optimizer - unet_lr: float = 5.0e-4 + unet_lr: float = 1.0e-3 prodigy_d_coef: float = 1.0 unet_prodigy_growth_factor: float = 1.05 # lower values make the lr go up slower (1.01 is for 1k step runs, 1.02 is for 500 step runs) lora_weight_decay: float = 0.002 @@ -39,8 +39,8 @@ class TrainingConfig(BaseModel): ti_optimizer: Literal["adamw", "prodigy"] = "adamw" freeze_ti_after_completion_f: float = 1.0 # freeze the TI after this fraction of the training is done - cond_reg_w: float = 2.0e-5 - tok_cond_reg_w: float = 1.0e-5 + cond_reg_w: float = 0.0e-5 + tok_cond_reg_w: float = 0.0e-5 tok_cov_reg_w: float = 2000. # regularizes the token covariance matrix wrt pretrained "healthy" tokens off_ratio_power: float = 0.02 # Pulls the std of the token distribution towards the target std l1_penalty: float = 0.01 # Makes the unet lora matrix more sparse @@ -85,7 +85,7 @@ class TrainingConfig(BaseModel): if text_encoder_lora_optimizer is not None then everything else is used. Else the other variables are ignored. """ - text_encoder_lora_optimizer: Union[None, Literal["adamw"]] = "adamw" + text_encoder_lora_optimizer: Union[None, Literal["adamw"]] = null text_encoder_lora_lr: float = 1.0e-5 txt_encoders_lr_warmup_steps: int = 200 text_encoder_lora_weight_decay: float = 1.0e-5 diff --git a/trainer/inference.py b/trainer/inference.py index e76c8ae..9f63b64 100644 --- a/trainer/inference.py +++ b/trainer/inference.py @@ -247,8 +247,11 @@ def encode_prompt_advanced( Helper function to encode the lora_prompt (containing a trained token) and a zero prompt (without the token) This allows interpolating the strength of the trained token in the final image. """ + if lora_path: + lora_prompt = prepare_prompt_for_lora(prompt, lora_path, verbose=1) + else: + lora_prompt = prompt - lora_prompt = prepare_prompt_for_lora(prompt, lora_path, verbose=1) if concept_mode == "face": replace_str = "person" elif concept_mode == "object": diff --git a/trainer/loss.py b/trainer/loss.py index 603afa8..2784f29 100644 --- a/trainer/loss.py +++ b/trainer/loss.py @@ -177,8 +177,16 @@ class ConditioningRegularizer: class DistributionLoss(torch.nn.Module): + """ + Initialized a new DistributionLoss with shape: torch.Size([49410, 768]) + Initialized a new DistributionLoss with shape: torch.Size([49410, 1280]) + + Class to simplify the calculation of the covariance loss between the trained token embeddings and the pretrained embeddings. + + """ def __init__(self, pretrained_embeddings, dtype=torch.float32, outdir = None): super(DistributionLoss, self).__init__() + print(f"Initialized a new DistributionLoss with shape: {pretrained_embeddings.shape}") self.dtype = dtype self.target_cov = self._calculate_covariance(pretrained_embeddings) self.target_stds = pretrained_embeddings.std(-1) diff --git a/training_args_face.json b/training_args_face.json index 8c52ef4..a7d9a69 100644 --- a/training_args_face.json +++ b/training_args_face.json @@ -1,27 +1,28 @@ { - "output_dir": "lora_models/kofi", - "sd_model_version": "sdxl", - "lora_training_urls": "/home/rednax/Documents/datasets/DOV/kofi", + "output_dir": "lora_models/xander_best", + "sd_model_version": "sd15", + "lora_training_urls": "/home/rednax/Documents/datasets/people/xander", "concept_mode": "face", "seed": 0, "resolution": 512, - "validation_img_size": [1024, 1024], "train_batch_size": 4, - "n_sample_imgs": 4, - "max_train_steps": 360, + "n_sample_imgs": 6, + "max_train_steps": 500, "token_warmup_steps": 0, - "checkpointing_steps": 120, + "checkpointing_steps": 100, "gradient_accumulation_steps": 1, "n_tokens": 2, "ti_lr": 0.001, "ti_weight_decay": 0.0005, - "text_encoder_lora_optimizer": "adamw", + "text_encoder_lora_optimizer": null, "text_encoder_lora_lr": 1.0e-4, "text_encoder_lora_weight_decay": 1e-5, "text_encoder_lora_rank": 12, + + "unet_lr": 0.001, "prodigy_d_coef": 1.0, "unet_prodigy_growth_factor": 1.05, - "lora_rank": 12, + "lora_rank": 16, "use_dora": false, "caption_model": "gpt4-v", "debug": true