From 37bcfe61c9b2c315286efdec7ec58bab3e680bf9 Mon Sep 17 00:00:00 2001 From: xander Date: Fri, 26 Apr 2024 08:45:29 +0200 Subject: [PATCH] hyperparam tweaks --- scripts/create_hyperparam_sweep.py | 26 ++++++++++++++------------ scripts/evaluate_gridsearch.py | 21 +++++++++++++++------ trainer/config.py | 14 +++++++------- training_args_face.json | 10 +++++----- training_args_style.json | 13 +++++++------ 5 files changed, 48 insertions(+), 36 deletions(-) diff --git a/scripts/create_hyperparam_sweep.py b/scripts/create_hyperparam_sweep.py index af7bd4a..d66d3f2 100644 --- a/scripts/create_hyperparam_sweep.py +++ b/scripts/create_hyperparam_sweep.py @@ -35,11 +35,11 @@ def hamming_distance(dict1, dict2): ####################################################################################### # Setup the base experiment config: -exp_name = "gridsearch_sdxl_beeple" +exp_name = "gridsearch_sdxl_styles" caption_prefix = "" mask_target_prompts = "" n_exp = 200 # how many random experiment settings to generate -min_hamming_distance = 2 # min_n_params that have to be different from any previous experiment to be scheduled +min_hamming_distance = 3 # min_n_params that have to be different from any previous experiment to be scheduled output_sh_path = f"gridsearch_configs/{exp_name}.sh" @@ -50,7 +50,9 @@ hyperparameters = { "output_dir": [f"lora_models/{exp_name}"], "sd_model_version": ["sdxl"], "lora_training_urls": [ - "/home/rednax/Documents/datasets/beeple" + "/home/rednax/Documents/datasets/sweep/beeple", + "/home/rednax/Documents/datasets/sweep/clipx_75", + "/home/rednax/Documents/datasets/sweep/speelveld" ], "concept_mode": ['style'], "seed": [0], @@ -58,21 +60,21 @@ hyperparameters = { "validation_img_size": [[1024, 1024]], "train_batch_size": [4], "n_sample_imgs": [6], - "max_train_steps": [600], - "checkpointing_steps": [100], + "max_train_steps": [400], + "checkpointing_steps": [80], "gradient_accumulation_steps": [1], - "freeze_ti_after_completion_f": [0.4,0.8], - "tok_cov_reg_w": [0.0, 0.001, 0.005, 0.015], - "text_encoder_lora_optimizer": ["adamw", None], - "prodigy_d_coef": [0.5,1.0], - "cond_reg_w": [0.0], - "tok_cond_reg_w": [0.0], + "token_warmup_steps": [100,0], + "freeze_ti_after_completion_f": [0.5,1.0], + "tok_cov_reg_w": [500,1000,2000], + "text_encoder_lora_optimizer": ["adamw"], + "text_encoder_lora_lr": [0.0, 0.5e-4, 2.0e-5], + "unet_prodigy_growth_factor": [1.05, 1.02, 1.01], + "prodigy_d_coef": [1.0], "n_tokens": [2,3], "ti_lr": [0.001], "ti_weight_decay": [0.0005], "lora_weight_decay": [0.001], "l1_penalty": [0.1, 0.0], - "off_ratio_power": [0.0, 0.05], "token_embedding_lr_warmup_steps": [0], "snr_gamma": [5.0], "lora_rank": [12,24], diff --git a/scripts/evaluate_gridsearch.py b/scripts/evaluate_gridsearch.py index 54c9711..75c4f8e 100644 --- a/scripts/evaluate_gridsearch.py +++ b/scripts/evaluate_gridsearch.py @@ -11,8 +11,11 @@ from sklearn.metrics import r2_score # Define paths -exp_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/lora_models/beeple_02" -config_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/gridsearch_configs/gridsearch_sdxl_beeple" +exp_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/lora_models/style_sweep/all" +config_dir = "/home/rednax/SSD2TB/Github_repos/diffusion_trainer/gridsearch_configs/gridsearch_sdxl_styles" + +output_dir = f"gridsearch_configs/results/{os.path.basename(config_dir)}" +output_suffix = f"{os.path.basename(exp_dir)}" # Initialize a dictionary to hold parameter values and associated scores parameters = defaultdict(lambda: defaultdict(list)) @@ -49,15 +52,21 @@ from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score from sklearn.preprocessing import LabelEncoder +os.makedirs(output_dir, exist_ok=True) + def plot_parameters(parameters): for param, data in parameters.items(): values = np.array(data['values']) scores = np.array(data['scores']) + + + noise_strength_values = 0.02 + noise_strength_scores = 0.02 # Determine if values are numeric if values.dtype.kind in 'bifc': # Numeric types # Add noise directly to values - jittered_values = values + np.random.normal(0, 0.01 * np.max(values), values.shape) + jittered_values = values + np.random.normal(0, noise_strength_values * (np.max(values) - np.min(values)), values.shape) else: # Encode string values to integers for plotting encoder = LabelEncoder() @@ -75,7 +84,7 @@ def plot_parameters(parameters): predicted_scores = model.predict(values_reshaped) # add some jitter to the scores: - jittered_scores = scores + np.random.normal(0, 0.02 * np.max(scores), scores.shape) + jittered_scores = scores + np.random.normal(0, noise_strength_scores * np.max(scores), scores.shape) # Calculate R² value r_squared = r2_score(scores, predicted_scores) @@ -84,7 +93,7 @@ def plot_parameters(parameters): sns.scatterplot(x=jittered_values, y=jittered_scores, alpha=0.6) # Plot trendline - sns.lineplot(x=np.sort(values), y=predicted_scores[np.argsort(values)], color='red', label=f'Fit: y={model.coef_[0]:.2f}x+{model.intercept_:.2f}, R²={r_squared:.2f}') + sns.lineplot(x=np.sort(values), y=predicted_scores[np.argsort(values)], color='red', label=f'Fit: y={model.coef_[0]:.1f}x+{model.intercept_:.1f}, R²={r_squared:.2f}') # Set plot title and labels plt.title(f'Influence of {param} on the score') @@ -93,7 +102,7 @@ def plot_parameters(parameters): plt.legend() # Save and close the plot - plt.savefig(f'res_{param}.png') + plt.savefig(f'{output_dir}/res_{param}_{output_suffix}.png') plt.close() # Call the updated function with your parameters dictionary diff --git a/trainer/config.py b/trainer/config.py index 38cd268..0ca9abc 100644 --- a/trainer/config.py +++ b/trainer/config.py @@ -19,9 +19,8 @@ class TrainingConfig(BaseModel): train_batch_size: int = 4 num_train_epochs: int = 10000 max_train_steps: int = 500 - token_warmup_steps: int = 40 + token_warmup_steps: int = 50 checkpointing_steps: int = 10000 - txt_encoders_lr_warmup_steps: int = 30 gradient_accumulation_steps: int = 1 is_lora: bool = True prodigy_d_coef: float = 1.0 @@ -29,13 +28,13 @@ class TrainingConfig(BaseModel): ti_lr: float = 1e-3 ti_weight_decay: float = 3e-4 ti_optimizer: Literal["adamw", "prodigy"] = "adamw" - freeze_ti_after_completion_f: float = 0.5 # freeze the TI after this fraction of the training is done + freeze_ti_after_completion_f: float = 1.0 # freeze the TI after this fraction of the training is done lora_weight_decay: float = 0.002 cond_reg_w: float = 0.0e-5 tok_cond_reg_w: float = 0.0e-5 - tok_cov_reg_w: float = 750. # regularizes the token covariance matrix wrt pretrained "healthy" tokens + tok_cov_reg_w: float = 1000. # regularizes the token covariance matrix wrt pretrained "healthy" tokens off_ratio_power: float = 0.01 # Pulls the std of the token distribution towards the target std - l1_penalty: float = 0.1 # Makes the unet lora matrix more sparse + l1_penalty: float = 0.01 # Makes the unet lora matrix more sparse noise_offset: float = 0.02 # Noise offset training to improve very dark / very bright images snr_gamma: float = 5.0 lora_alpha_multiplier: float = 1.0 @@ -79,8 +78,9 @@ class TrainingConfig(BaseModel): Else the other variables are ignored. """ text_encoder_lora_optimizer: Union[None, Literal["adamw"]] = "adamw" - text_encoder_lora_lr: float = 0.5e-5 - text_encoder_lora_weight_decay: float = 1e-5 + text_encoder_lora_lr: float = 1.5e-5 + txt_encoders_lr_warmup_steps: int = 100 + text_encoder_lora_weight_decay: float = 1.0e-5 text_encoder_lora_rank: int = 12 def __init__(self, **data): diff --git a/training_args_face.json b/training_args_face.json index b6e9f00..e80f3df 100644 --- a/training_args_face.json +++ b/training_args_face.json @@ -1,7 +1,7 @@ { - "output_dir": "lora_models/xander_ti", + "output_dir": "lora_models/mira_txt_lora_only_2.0_lr", "sd_model_version": "sdxl", - "lora_training_urls": "https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/xander.zip", + "lora_training_urls": "/home/rednax/Documents/datasets/people/mira1", "concept_mode": "face", "seed": 0, "resolution": 512, @@ -14,10 +14,10 @@ "gradient_accumulation_steps": 1, "n_tokens": 2, "ti_lr": 0.001, - "freeze_ti_after_completion_f": 1.0, + "freeze_ti_after_completion_f": 0.0, "ti_weight_decay": 0.0005, - "text_encoder_lora_optimizer": null, - "text_encoder_lora_lr": 0.5e-4, + "text_encoder_lora_optimizer": "adamw", + "text_encoder_lora_lr": 2.0e-4, "text_encoder_lora_weight_decay": 1e-5, "text_encoder_lora_rank": 12, "off_ratio_power": 0.02, diff --git a/training_args_style.json b/training_args_style.json index b5d6099..19b85fb 100644 --- a/training_args_style.json +++ b/training_args_style.json @@ -1,18 +1,19 @@ { - "output_dir": "lora_models/clipx", + "output_dir": "lora_models/does_3_tokens_higher_cov_reg", "sd_model_version": "sdxl", - "lora_training_urls": "https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/clipx.zip", + "lora_training_urls": "https://storage.googleapis.com/public-assets-xander/A_workbox/lora_training_sets/does.zip", "concept_mode": "style", "seed": 0, "resolution": 512, "validation_img_size": [1024, 1024], - "train_batch_size": 4, - "n_sample_imgs": 4, + "train_batch_size": 3, + "n_sample_imgs": 6, "max_train_steps": 400, "token_warmup_steps": 100, "checkpointing_steps": 80, "gradient_accumulation_steps": 1, - "n_tokens": 2, + "n_tokens": 4, + "tok_cov_reg_w": 1500, "ti_lr": 0.001, "freeze_ti_after_completion_f": 1.0, "ti_weight_decay": 0.0005, @@ -29,7 +30,7 @@ "lora_alpha_multiplier": 1.0, "lora_rank": 12, "use_dora": false, - "caption_model": "gpt4-v", + "caption_model": "blip", "verbose": true, "debug": true } \ No newline at end of file