Integrate CharacterGen & Improve its workflow with Unique3D

2024-07-09 14:54:44 -06:00
parent a805a5a6eb
commit a4fb6cfa68
102 changed files with 20754 additions and 271 deletions
@@ -0,0 +1,23 @@
+pretrained_model_path: "stabilityai/stable-diffusion-2-1"
+image_encoder_path: "./2D_Stage/models/image_encoder"
+ckpt_dir: "./2D_Stage/models/checkpoint"
+
+validation:
+  use_inv_latent: False
+  video_length: 4
+
+use_pose_guider: True
+use_noise: False
+use_shifted_noise: False
+unet_condition_type: image
+
+unet_from_pretrained_kwargs:
+  camera_embedding_type: 'e_de_da_sincos'
+  projection_class_embeddings_input_dim: 10  # modify
+  joint_attention: false  # modify
+  num_views: 4
+  sample_size: 96
+  zero_init_conv_in: false
+  zero_init_camera_projection: false  
+  in_channels: 4
+  use_safetensors: true
@@ -0,0 +1,104 @@
+system_cls: CharacterGen.Stage_3D.lrm.systems.multiview_lrm.MultiviewLRM
+data:
+  cond_width: 504
+  cond_height: 504
+
+system:
+  weights: 3D_Stage/models/lrm.ckpt
+
+  weights_ignore_modules:
+    - decoder.heads.density
+
+  check_train_every_n_steps: 100
+
+  camera_embedder_cls: CharacterGen.Stage_3D.lrm.models.camera.LinearCameraEmbedder
+  camera_embedder:
+    in_channels: 16
+    out_channels: 768
+    conditions:
+      - c2w_cond
+
+  # image tokenizer transforms input images to tokens
+  image_tokenizer_cls: CharacterGen.Stage_3D.lrm.models.tokenizers.image.DINOV2SingleImageTokenizer
+  image_tokenizer:
+    pretrained_model_name_or_path: "3D_Stage/models/base"
+    freeze_backbone_params: false
+    enable_memory_efficient_attention: true
+    enable_gradient_checkpointing: true
+    # camera modulation to the DINO transformer layers
+    modulation: true
+    modulation_zero_init: true
+    modulation_single_layer: true
+    modulation_cond_dim: ${system.camera_embedder.out_channels}
+
+  # tokenizer gives a tokenized representation for the 3D scene
+  # triplane tokens in this case
+  tokenizer_cls: CharacterGen.Stage_3D.lrm.models.tokenizers.triplane.TriplaneLearnablePositionalEmbedding
+  tokenizer:
+    plane_size: 32
+    num_channels: 512
+
+  # backbone network is a transformer that takes scene tokens (potentially with conditional image tokens)
+  # and outputs scene tokens of the same size
+  backbone_cls: CharacterGen.Stage_3D.lrm.models.transformers.transformer_1d.Transformer1D
+  backbone:
+    in_channels: ${system.tokenizer.num_channels}
+    num_attention_heads: 16
+    attention_head_dim: 64
+    num_layers: 12
+    cross_attention_dim: 768 # hard-code, =DINO feature dim
+    # camera modulation to the transformer layers
+    # if not needed, set norm_type=layer_norm and do not specify cond_dim_ada_norm_continuous
+    norm_type: "layer_norm"
+    enable_memory_efficient_attention: true
+    gradient_checkpointing: true
+
+  # post processor takes scene tokens and outputs the final scene parameters that will be used for rendering
+  # in this case, triplanes are upsampled and the features are condensed
+  post_processor_cls: CharacterGen.Stage_3D.lrm.models.networks.TriplaneUpsampleNetwork
+  post_processor:
+    in_channels: 512
+    out_channels: 80
+
+  renderer_cls: CharacterGen.Stage_3D.lrm.models.renderers.triplane_dmtet.TriplaneDMTetRenderer
+  renderer:
+    radius: 0.6 # slightly larger than 0.5
+    feature_reduction: concat
+    sdf_bias: -2.
+    tet_dir: "tets"
+    isosurface_resolution: 256
+    enable_isosurface_grid_deformation: false
+    sdf_activation: negative
+
+  decoder_cls: CharacterGen.Stage_3D.lrm.models.networks.MultiHeadMLP
+  decoder:
+    in_channels: 240 # 3 * 80
+    n_neurons: 64
+    n_hidden_layers_share: 8
+    heads:
+      - name: sdf
+        out_channels: 1
+        n_hidden_layers: 1
+        output_activation: null
+      - name: features
+        out_channels: 3
+        n_hidden_layers: 1
+        output_activation: null # activate in material
+    activation: silu
+    chunk_mode: deferred
+    chunk_size: 131072
+
+  exporter:
+    fmt: "obj"
+   #visual: "vertex"
+    visual: "uv"
+    save_uv: True
+    save_texture: True
+    uv_unwrap_method: "open3d"
+    output_path: "3D_Stage/outputs"
+
+  material_cls: CharacterGen.Stage_3D.lrm.models.materials.no_material.NoMaterial
+
+  background_cls: CharacterGen.Stage_3D.lrm.models.background.solid_color_background.SolidColorBackground
+  background:
+    color: [0.5, 0.5, 0.5]