Integrated TripoSG plus Scribble model
This commit is contained in:
Mr. For Example
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
license: mit
|
||||
pipeline_tag: image-to-3d
|
||||
tags:
|
||||
- triposg
|
||||
- 3d-generation
|
||||
- rectified-flow
|
||||
---
|
||||
# TripoSG-scribble - Fast 3D Shape Prototyping with Scribble and Prompt
|
||||
|
||||
TripoSG-scribble converts a scribble image and a text prompt to a 3D shape. TripoSG-scribble is a variant of TripoSG. TripoSG is a state-of-the-art image-to-3D generation foundation model that leverages large-scale rectified flow transformers to produce high-fidelity 3D shapes from single images.
|
||||
|
||||
## Model Description
|
||||
|
||||
### Model Architecture
|
||||
|
||||
TripoSG utilizes a novel architecture combining:
|
||||
- Rectified Flow (RF) based Transformer for stable, linear trajectory modeling
|
||||
- Advanced VAE with SDF-based representation and hybrid geometric supervision
|
||||
- Cross-attention mechanism for image feature condition
|
||||
- 1.5B parameters operating on 2048 latent tokens
|
||||
|
||||
For inference efficiency, TripoSG-scribble is different from TripoSG in:
|
||||
- TripoSG-scribble is a CFG-distilled model and should be used with CFG=0
|
||||
- TripoSG-scribble is trained with 512 latent tokens
|
||||
|
||||
## Intended Uses
|
||||
|
||||
This model is designed for:
|
||||
- Converting scribble image and text prompt to high-quality 3D meshes
|
||||
- Creative and design applications
|
||||
- Gaming and VFX asset creation
|
||||
- Prototyping and visualization
|
||||
|
||||
## Requirements
|
||||
|
||||
- CUDA-capable GPU (>8GB VRAM)
|
||||
|
||||
## Usage
|
||||
|
||||
For detailed usage instructions, please visit our [GitHub repository](https://github.com/VAST-AI-Research/TripoSG).
|
||||
|
||||
## About
|
||||
|
||||
TripoSG-scribble is developed by [Tripo](https://www.tripo3d.ai), [VAST AI Research](https://github.com/orgs/VAST-AI-Research), pushing the boundaries of 3D Generative AI.
|
||||
For more information:
|
||||
- [GitHub Repository](https://github.com/VAST-AI-Research/TripoSG)
|
||||
- [Paper](https://arxiv.org/abs/2502.06608)
|
||||
- [Gradio Demo](https://huggingface.co/spaces/VAST-AI/TripoSG-scribble)
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"crop_size": {
|
||||
"height": 224,
|
||||
"width": 224
|
||||
},
|
||||
"do_center_crop": true,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"image_mean": [
|
||||
0.485,
|
||||
0.456,
|
||||
0.406
|
||||
],
|
||||
"image_processor_type": "BitImageProcessor",
|
||||
"image_std": [
|
||||
0.229,
|
||||
0.224,
|
||||
0.225
|
||||
],
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"size": {
|
||||
"shortest_edge": 256
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"apply_layernorm": true,
|
||||
"architectures": [
|
||||
"Dinov2Model"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.0,
|
||||
"drop_path_rate": 0.0,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.0,
|
||||
"hidden_size": 1024,
|
||||
"image_size": 518,
|
||||
"initializer_range": 0.02,
|
||||
"layer_norm_eps": 1e-06,
|
||||
"layerscale_value": 1.0,
|
||||
"mlp_ratio": 4,
|
||||
"model_type": "dinov2",
|
||||
"num_attention_heads": 16,
|
||||
"num_channels": 3,
|
||||
"num_hidden_layers": 24,
|
||||
"out_features": [
|
||||
"stage24"
|
||||
],
|
||||
"out_indices": [
|
||||
24
|
||||
],
|
||||
"patch_size": 14,
|
||||
"qkv_bias": true,
|
||||
"reshape_hidden_states": true,
|
||||
"stage_names": [
|
||||
"stem",
|
||||
"stage1",
|
||||
"stage2",
|
||||
"stage3",
|
||||
"stage4",
|
||||
"stage5",
|
||||
"stage6",
|
||||
"stage7",
|
||||
"stage8",
|
||||
"stage9",
|
||||
"stage10",
|
||||
"stage11",
|
||||
"stage12",
|
||||
"stage13",
|
||||
"stage14",
|
||||
"stage15",
|
||||
"stage16",
|
||||
"stage17",
|
||||
"stage18",
|
||||
"stage19",
|
||||
"stage20",
|
||||
"stage21",
|
||||
"stage22",
|
||||
"stage23",
|
||||
"stage24"
|
||||
],
|
||||
"torch_dtype": "float16",
|
||||
"transformers_version": "4.50.3",
|
||||
"use_mask_token": true,
|
||||
"use_swiglu_ffn": false
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"_class_name": "TripoSGScribblePipeline",
|
||||
"_diffusers_version": "0.33.1",
|
||||
"feature_extractor_dinov2": [
|
||||
"transformers",
|
||||
"BitImageProcessor"
|
||||
],
|
||||
"image_encoder_dinov2": [
|
||||
"transformers",
|
||||
"Dinov2Model"
|
||||
],
|
||||
"scheduler": [
|
||||
"TripoSG.schedulers.scheduling_rectified_flow",
|
||||
"RectifiedFlowScheduler"
|
||||
],
|
||||
"text_encoder": [
|
||||
"transformers",
|
||||
"CLIPTextModelWithProjection"
|
||||
],
|
||||
"tokenizer": [
|
||||
"transformers",
|
||||
"CLIPTokenizer"
|
||||
],
|
||||
"transformer": [
|
||||
"TripoSG.models.transformers.triposg_transformer",
|
||||
"TripoSGDiTModel"
|
||||
],
|
||||
"vae": [
|
||||
"TripoSG.models.autoencoders.autoencoder_kl_triposg",
|
||||
"TripoSGVAEModel"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"_class_name": "RectifiedFlowScheduler",
|
||||
"_diffusers_version": "0.33.1",
|
||||
"num_train_timesteps": 1000,
|
||||
"shift": 1,
|
||||
"use_dynamic_shifting": false
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"architectures": [
|
||||
"CLIPTextModelWithProjection"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 0,
|
||||
"dropout": 0.0,
|
||||
"eos_token_id": 2,
|
||||
"hidden_act": "quick_gelu",
|
||||
"hidden_size": 768,
|
||||
"initializer_factor": 1.0,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 3072,
|
||||
"layer_norm_eps": 1e-05,
|
||||
"max_position_embeddings": 77,
|
||||
"model_type": "clip_text_model",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 12,
|
||||
"pad_token_id": 1,
|
||||
"projection_dim": 768,
|
||||
"torch_dtype": "float16",
|
||||
"transformers_version": "4.50.3",
|
||||
"vocab_size": 49408
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<|startoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"49406": {
|
||||
"content": "<|startoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"49407": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"bos_token": "<|startoftext|>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"do_lower_case": true,
|
||||
"eos_token": "<|endoftext|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 77,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"tokenizer_class": "CLIPTokenizer",
|
||||
"unk_token": "<|endoftext|>"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"_class_name": "TripoSGDiTModel",
|
||||
"_diffusers_version": "0.33.1",
|
||||
"cross_attention_2_dim": 1024,
|
||||
"cross_attention_dim": 768,
|
||||
"in_channels": 64,
|
||||
"num_attention_heads": 16,
|
||||
"num_layers": 21,
|
||||
"use_cross_attention_2": true,
|
||||
"width": 2048
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"_class_name": "TripoSGVAEModel",
|
||||
"_diffusers_version": "0.33.1",
|
||||
"embed_frequency": 8,
|
||||
"embed_include_pi": false,
|
||||
"embedding_type": "frequency",
|
||||
"in_channels": 3,
|
||||
"latent_channels": 64,
|
||||
"num_attention_heads": 8,
|
||||
"num_layers_decoder": 16,
|
||||
"num_layers_encoder": 8,
|
||||
"width_decoder": 1024,
|
||||
"width_encoder": 512
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.gz filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -0,0 +1,45 @@
|
||||
---
|
||||
license: mit
|
||||
pipeline_tag: image-to-3d
|
||||
tags:
|
||||
- triposg
|
||||
- 3d-generation
|
||||
- rectified-flow
|
||||
---
|
||||
# TripoSG - High-Fidelity 3D Shape Synthesis using Large-Scale Rectified Flow Models
|
||||
|
||||
TripoSG is a state-of-the-art image-to-3D generation foundation model that leverages large-scale rectified flow transformers to produce high-fidelity 3D shapes from single images.
|
||||
|
||||
## Model Description
|
||||
|
||||
### Model Architecture
|
||||
|
||||
TripoSG utilizes a novel architecture combining:
|
||||
- Rectified Flow (RF) based Transformer for stable, linear trajectory modeling
|
||||
- Advanced VAE with SDF-based representation and hybrid geometric supervision
|
||||
- Cross-attention mechanism for image feature condition
|
||||
- 1.5B parameters operating on 2048 latent tokens
|
||||
|
||||
## Intended Uses
|
||||
|
||||
This model is designed for:
|
||||
- Converting single images to high-quality 3D meshes
|
||||
- Creative and design applications
|
||||
- Gaming and VFX asset creation
|
||||
- Prototyping and visualization
|
||||
|
||||
## Requirements
|
||||
|
||||
- CUDA-capable GPU (>8GB VRAM)
|
||||
|
||||
## Usage
|
||||
|
||||
For detailed usage instructions, please visit our [GitHub repository](https://github.com/VAST-AI-Research/TripoSG).
|
||||
|
||||
## About
|
||||
|
||||
TripoSG is developed by [Tripo](https://www.tripo3d.ai), [VAST AI Research](https://github.com/orgs/VAST-AI-Research), pushing the boundaries of 3D Generative AI.
|
||||
For more information:
|
||||
- [GitHub Repository](https://github.com/VAST-AI-Research/TripoSG)
|
||||
- [Paper](https://arxiv.org/abs/2502.06608)
|
||||
- [Gradio Demo](https://huggingface.co/spaces/VAST-AI/TripoSG)
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"crop_size": {
|
||||
"height": 224,
|
||||
"width": 224
|
||||
},
|
||||
"do_center_crop": true,
|
||||
"do_convert_rgb": true,
|
||||
"do_normalize": true,
|
||||
"do_rescale": true,
|
||||
"do_resize": true,
|
||||
"image_mean": [
|
||||
0.485,
|
||||
0.456,
|
||||
0.406
|
||||
],
|
||||
"image_processor_type": "BitImageProcessor",
|
||||
"image_std": [
|
||||
0.229,
|
||||
0.224,
|
||||
0.225
|
||||
],
|
||||
"resample": 3,
|
||||
"rescale_factor": 0.00392156862745098,
|
||||
"size": {
|
||||
"shortest_edge": 256
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"_name_or_path": "facebook/dinov2-large",
|
||||
"apply_layernorm": true,
|
||||
"architectures": [
|
||||
"Dinov2Model"
|
||||
],
|
||||
"attention_probs_dropout_prob": 0.0,
|
||||
"drop_path_rate": 0.0,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout_prob": 0.0,
|
||||
"hidden_size": 1024,
|
||||
"image_size": 518,
|
||||
"initializer_range": 0.02,
|
||||
"layer_norm_eps": 1e-06,
|
||||
"layerscale_value": 1.0,
|
||||
"mlp_ratio": 4,
|
||||
"model_type": "dinov2",
|
||||
"num_attention_heads": 16,
|
||||
"num_channels": 3,
|
||||
"num_hidden_layers": 24,
|
||||
"out_features": [
|
||||
"stage24"
|
||||
],
|
||||
"out_indices": [
|
||||
24
|
||||
],
|
||||
"patch_size": 14,
|
||||
"qkv_bias": true,
|
||||
"reshape_hidden_states": true,
|
||||
"stage_names": [
|
||||
"stem",
|
||||
"stage1",
|
||||
"stage2",
|
||||
"stage3",
|
||||
"stage4",
|
||||
"stage5",
|
||||
"stage6",
|
||||
"stage7",
|
||||
"stage8",
|
||||
"stage9",
|
||||
"stage10",
|
||||
"stage11",
|
||||
"stage12",
|
||||
"stage13",
|
||||
"stage14",
|
||||
"stage15",
|
||||
"stage16",
|
||||
"stage17",
|
||||
"stage18",
|
||||
"stage19",
|
||||
"stage20",
|
||||
"stage21",
|
||||
"stage22",
|
||||
"stage23",
|
||||
"stage24"
|
||||
],
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.45.2",
|
||||
"use_swiglu_ffn": false
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"_class_name": "TripoSGPipeline",
|
||||
"_diffusers_version": "0.30.3",
|
||||
"feature_extractor_dinov2": [
|
||||
"transformers",
|
||||
"BitImageProcessor"
|
||||
],
|
||||
"image_encoder_dinov2": [
|
||||
"transformers",
|
||||
"Dinov2Model"
|
||||
],
|
||||
"scheduler": [
|
||||
"TripoSG.schedulers.scheduling_rectified_flow",
|
||||
"RectifiedFlowScheduler"
|
||||
],
|
||||
"transformer": [
|
||||
"TripoSG.models.transformers.triposg_transformer",
|
||||
"TripoSGDiTModel"
|
||||
],
|
||||
"vae": [
|
||||
"TripoSG.models.autoencoders.autoencoder_kl_triposg",
|
||||
"TripoSGVAEModel"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"_class_name": "RectifiedFlowScheduler",
|
||||
"_diffusers_version": "0.30.3",
|
||||
"num_train_timesteps": 1000,
|
||||
"shift": 1,
|
||||
"use_dynamic_shifting": false
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_class_name": "TripoSGDiTModel",
|
||||
"_diffusers_version": "0.30.3",
|
||||
"cross_attention_dim": 1024,
|
||||
"in_channels": 64,
|
||||
"num_attention_heads": 16,
|
||||
"num_layers": 21,
|
||||
"width": 2048
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"_class_name": "Tripo2VAEModel",
|
||||
"_diffusers_version": "0.30.3",
|
||||
"embed_frequency": 8,
|
||||
"embed_include_pi": false,
|
||||
"embedding_type": "frequency",
|
||||
"in_channels": 3,
|
||||
"latent_channels": 64,
|
||||
"num_attention_heads": 8,
|
||||
"num_layers_decoder": 16,
|
||||
"num_layers_encoder": 8,
|
||||
"width_decoder": 1024,
|
||||
"width_encoder": 512
|
||||
}
|
||||
Reference in New Issue
Block a user