import argparse import json import os import sys from types import SimpleNamespace import torch class SimpleNamespaceCNWarrper(SimpleNamespace): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__dict__.update(kwargs) self.__iter__ = lambda: iter(kwargs.keys()) # is not iterable def __iter__(self): return iter(self.__dict__.keys()) # object has no attribute 'num_attention_heads' def __getattr__(self, name): return self.__dict__.get(name, None) if __name__ == "__main__": parser = argparse.ArgumentParser( conflict_handler='resolve', ) parser.add_argument("--sys_path", type=str, default="") parser.add_argument("--train_config_file", type=str, default="") parser.add_argument("--mz_master_port", type=int, default=0) args = parser.parse_args() master_port = args.mz_master_port print(f"master_port = {master_port}") try: from . import hook_HYDiT_utils except Exception as e: import hook_HYDiT_utils hook_HYDiT_utils.set_master_port(master_port) sys_path = args.sys_path if sys_path != "": sys.path.append(sys_path) print("HYDi run hook") try: from . import hook_HYDiT_main_train_deepspeed except Exception as e: import hook_HYDiT_main_train_deepspeed import hydit.config def _handle_conflict_error(self, *args, **kwargs): pass def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) return args train_config_file = args.train_config_file if train_config_file == "": raise ValueError("train_config_file is empty") train_config = {} with open(train_config_file, "r") as f: train_config = json.load(f) argparse.ArgumentParser._handle_conflict_error = _handle_conflict_error argparse.ArgumentParser._handle_conflict_resolve = _handle_conflict_error argparse.ArgumentParser.parse_args = parse_args margs = hydit.config.get_args() margs.model = train_config.get("model", "DiT-g/2") margs.task_flag = train_config.get("task_flag") margs.resume_split = train_config.get("resume_split", True) margs.ema_to_module = train_config.get("ema_to_module", True) margs.deepspeed = False margs.predict_type = train_config.get("predict_type", "v_prediction") margs.training_parts = train_config.get("training_parts", "lora") margs.batch_size = train_config.get("batch_size", 1) margs.grad_accu_steps = train_config.get("grad_accu_steps", 1) margs.global_seed = train_config.get("global_seed", 0) margs.use_flash_attn = train_config.get("use_flash_attn", False) margs.use_fp16 = train_config.get("use_fp16", True) margs.qk_norm = train_config.get("qk_norm", True) margs.ema_dtype = train_config.get("ema_dtype", "fp32") margs.async_ema = False margs.ckpt_latest_every = 0x7fffffff margs.multireso = train_config.get("multireso", True) margs.epochs = train_config.get("epochs", 50) margs.target_ratios = train_config.get( "target_ratios", ['1:1', '3:4', '4:3', '16:9', '9:16']) margs.rope_img = train_config.get("rope_img", "base1024") margs.image_size = train_config.get("image_size", 1024) margs.rope_real = train_config.get("rope_real", True) margs.index_file = train_config.get("index_file", None) margs.lr = train_config.get("lr", 1e-5) margs.rank = train_config.get("rank", 8) margs.noise_offset = train_config.get("noise_offset", 0.0) margs.log_every = train_config.get("log_every", 99999999999999) margs.use_zero_stage = train_config.get("use_zero_stage", 2) margs.global_batch_size = train_config.get("global_batch_size", 1) margs.deepspeed = True margs.results_dir = train_config.get("results_dir") margs.mse_loss_weight_type = train_config.get( "mse_loss_weight_type", "constant") for k, v in train_config.items(): if hasattr(margs, k): setattr(margs, k, v) hook_HYDiT_utils.set_unet_path( train_config.get("unet_path")) hook_HYDiT_utils.set_vae_ema_path( train_config.get("vae_ema_path")) hook_HYDiT_utils.set_text_encoder_path( train_config.get("text_encoder_path")) hook_HYDiT_utils.set_tokenizer_path( train_config.get("tokenizer_path")) hook_HYDiT_utils.set_t5_encoder_path( train_config.get("t5_encoder_path")) hook_HYDiT_utils.set_train_config(train_config) try: # deepspeed/runtime/engine import deepspeed.runtime.engine deepspeed.runtime.engine.DeepSpeedEngine._do_sanity_check = lambda x: None except Exception as e: pass if type(margs.image_size) == int: margs.image_size = [margs.image_size, margs.image_size] hook_HYDiT_main_train_deepspeed.Core(margs)