+
InstantCharacter: Personalize Any Characters with a Scalable Diffusion Transformer Framework
+
+
+
+
+
+[**Jiale Tao**](https://github.com/JialeTao)
1 ·
+[**Yanbing Zhang**](https://github.com/Monalissaa)
1 ·
+[**Qixun Wang**](https://github.com/wangqixun)
12✝ ·
+[**Yiji Cheng**](https://www.linkedin.com/in/yiji-cheng-a8b922213/)
1 ·
+[**Haofan Wang**](https://haofanwang.github.io/)
2 ·
+[**Xu Bai**](https://huggingface.co/baymin0220)
2 ·
+Zhengguang Zhou
12 ·
+[**Ruihuang Li**](https://scholar.google.com/citations?user=8CfyOtQAAAAJ&hl=zh-CN)
1 ·
+[**Linqing Wang**](https://scholar.google.com/citations?user=Hy12lcEAAAAJ&hl=en)
12 · Chunyu Wang
1 ·
+Qin Lin
1 ·
+Qinglin Lu
1*
+
+
+
1Hunyuan, Tencent ·
2InstantX Team
+
+
✝tech lead ·
*corresponding authors
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+## Release
+- [2025/04/18] 🔥 We release the [demo](https://huggingface.co/spaces/InstantX/InstantCharacter) [checkpoints](https://huggingface.co/InstantX/InstantCharacter/) and [code](https://github.com/Tencent/InstantCharacter).
+
+- [2025/04/02] 🔥 We launch the [project page](https://instantcharacter.github.io/).
+
+
+## Download
+
+You can directly download the model from [Huggingface](https://huggingface.co/InstantX/InstantCharacter).
+```shell
+huggingface-cli download --resume-download Tencent/InstantCharacter --local-dir checkpoints --local-dir-use-symlinks False
+```
+
+If you cannot access to Huggingface, you can use [hf-mirror](https://hf-mirror.com/) to download models.
+```shell
+export HF_ENDPOINT=https://hf-mirror.com
+huggingface-cli download --resume-download Tencent/InstantCharacter --local-dir checkpoints --local-dir-use-symlinks False
+```
+
+Once you have prepared all models, the folder tree should be like:
+
+```
+ .
+ ├── assets
+ ├── checkpoints
+ ├── models
+ ├── infer_demo.py
+ ├── pipeline.py
+ └── README.md
+```
+
+
+## Usage
+
+
+```python
+# !pip install transformers accelerate diffusers huggingface_cli
+import torch
+from PIL import Image
+from pipeline import InstantCharacterFluxPipeline
+
+# Step 1 Load base model and adapter
+ip_adapter_path = 'checkpoints/instantcharacter_ip-adapter.bin'
+base_model = 'black-forest-labs/FLUX.1-dev'
+image_encoder_path = 'google/siglip-so400m-patch14-384'
+image_encoder_2_path = 'facebook/dinov2-giant'
+seed = 123456
+pipe = InstantCharacterFluxPipeline.from_pretrained(base_model, torch_dtype=torch.bfloat16)
+pipe.to("cuda")
+pipe.init_adapter(
+ image_encoder_path=image_encoder_path,
+ image_encoder_2_path=image_encoder_2_path,
+ subject_ipadapter_cfg=dict(subject_ip_adapter_path=ip_adapter_path, nb_token=1024),
+)
+
+# Step 2 Load reference image
+ref_image_path = 'assets/girl.jpg' # white background
+ref_image = Image.open(ref_image_path).convert('RGB')
+
+# Step 3 Inference without style
+prompt = "A girl is playing a guitar in street"
+image = pipe(
+ prompt=prompt,
+ num_inference_steps=28,
+ guidance_scale=3.5,
+ subject_image=ref_image,
+ subject_scale=0.9,
+ generator=torch.manual_seed(seed),
+).images[0]
+image.save("flux_instantcharacter.png")
+```
+
+
+You can use style lora
+
+
+```shell
+# download style lora
+huggingface-cli download --resume-download InstantX/FLUX.1-dev-LoRA-Ghibli --local-dir checkpoints/style_lora/ --local-dir-use-symlinks False
+huggingface-cli download --resume-download InstantX/FLUX.1-dev-LoRA-Makoto-Shinkai --local-dir checkpoints/style_lora/ --local-dir-use-symlinks False
+```
+
+```python
+# You can also use other style lora
+
+# Step 3 Inference with style
+lora_file_path = 'checkpoints/style_lora/ghibli_style.safetensors'
+trigger = 'ghibli style'
+prompt = "A girl is playing a guitar in street"
+image = pipe.with_style_lora(
+ lora_file_path=lora_file_path,
+ trigger=trigger,
+ prompt=prompt,
+ num_inference_steps=28,
+ guidance_scale=3.5,
+ subject_image=ref_image,
+ subject_scale=0.9,
+ generator=torch.manual_seed(seed),
+).images[0]
+image.save("flux_instantcharacter_style_ghibli.png")
+
+
+# Step 3 Inference with style
+lora_file_path = 'checkpoints/style_lora/Makoto_Shinkai_style.safetensors'
+trigger = 'Makoto Shinkai style'
+prompt = "A girl is playing a guitar in street"
+image = pipe.with_style_lora(
+ lora_file_path=lora_file_path,
+ trigger=trigger,
+ prompt=prompt,
+ num_inference_steps=28,
+ guidance_scale=3.5,
+ subject_image=ref_image,
+ subject_scale=0.9,
+ generator=torch.manual_seed(seed),
+).images[0]
+image.save("flux_instantcharacter_style_Makoto.png")
+```
+
+## More case
+Animal character are relatively unstable.
+
+
+
+
+
+
+
+
+
+
+
+
+## Acknowledgment
+ - Our work is sponsored by [HuggingFace](https://huggingface.co) and [fal.ai](https://fal.ai).
+
+
+
+
+
+
+
+
diff --git a/InstantCharacter/app.py b/InstantCharacter/app.py
new file mode 100644
index 0000000..2ac448b
--- /dev/null
+++ b/InstantCharacter/app.py
@@ -0,0 +1,299 @@
+import torch
+import random
+import numpy as np
+from PIL import Image
+
+# import gradio as gr
+from huggingface_hub import hf_hub_download
+from transformers import AutoModelForImageSegmentation
+from torchvision import transforms
+
+from .pipeline import InstantCharacterFluxPipeline
+
+# global variable
+MAX_SEED = np.iinfo(np.int32).max
+device = "cuda" if torch.cuda.is_available() else "cpu"
+dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
+
+# pre-trained weights
+ip_adapter_path = hf_hub_download(repo_id="Tencent/InstantCharacter", filename="instantcharacter_ip-adapter.bin")
+base_model = 'black-forest-labs/FLUX.1-dev'
+image_encoder_path = 'google/siglip-so400m-patch14-384'
+image_encoder_2_path = 'facebook/dinov2-giant'
+birefnet_path = 'ZhengPeng7/BiRefNet'
+makoto_style_lora_path = hf_hub_download(repo_id="InstantX/FLUX.1-dev-LoRA-Makoto-Shinkai", filename="Makoto_Shinkai_style.safetensors")
+ghibli_style_lora_path = hf_hub_download(repo_id="InstantX/FLUX.1-dev-LoRA-Ghibli", filename="ghibli_style.safetensors")
+
+# init InstantCharacter pipeline
+pipe = InstantCharacterFluxPipeline.from_pretrained(base_model, torch_dtype=torch.bfloat16)
+pipe.to(device)
+
+# load InstantCharacter
+pipe.init_adapter(
+ image_encoder_path=image_encoder_path,
+ image_encoder_2_path=image_encoder_2_path,
+ subject_ipadapter_cfg=dict(subject_ip_adapter_path=ip_adapter_path, nb_token=1024),
+)
+
+# load matting model
+birefnet = AutoModelForImageSegmentation.from_pretrained(birefnet_path, trust_remote_code=True)
+birefnet.to('cuda')
+birefnet.eval()
+birefnet_transform_image = transforms.Compose([
+ transforms.Resize((1024, 1024)),
+ transforms.ToTensor(),
+ transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+
+
+def remove_bkg(subject_image):
+
+ def infer_matting(img_pil):
+ input_images = birefnet_transform_image(img_pil).unsqueeze(0).to('cuda')
+
+ with torch.no_grad():
+ preds = birefnet(input_images)[-1].sigmoid().cpu()
+ pred = preds[0].squeeze()
+ pred_pil = transforms.ToPILImage()(pred)
+ mask = pred_pil.resize(img_pil.size)
+ mask = np.array(mask)
+ mask = mask[..., None]
+ return mask
+
+ def get_bbox_from_mask(mask, th=128):
+ height, width = mask.shape[:2]
+ x1, y1, x2, y2 = 0, 0, width - 1, height - 1
+
+ sample = np.max(mask, axis=0)
+ for idx in range(width):
+ if sample[idx] >= th:
+ x1 = idx
+ break
+
+ sample = np.max(mask[:, ::-1], axis=0)
+ for idx in range(width):
+ if sample[idx] >= th:
+ x2 = width - 1 - idx
+ break
+
+ sample = np.max(mask, axis=1)
+ for idx in range(height):
+ if sample[idx] >= th:
+ y1 = idx
+ break
+
+ sample = np.max(mask[::-1], axis=1)
+ for idx in range(height):
+ if sample[idx] >= th:
+ y2 = height - 1 - idx
+ break
+
+ x1 = np.clip(x1, 0, width-1).round().astype(np.int32)
+ y1 = np.clip(y1, 0, height-1).round().astype(np.int32)
+ x2 = np.clip(x2, 0, width-1).round().astype(np.int32)
+ y2 = np.clip(y2, 0, height-1).round().astype(np.int32)
+
+ return [x1, y1, x2, y2]
+
+ def pad_to_square(image, pad_value = 255, random = False):
+ '''
+ image: np.array [h, w, 3]
+ '''
+ H,W = image.shape[0], image.shape[1]
+ if H == W:
+ return image
+
+ padd = abs(H - W)
+ if random:
+ padd_1 = int(np.random.randint(0,padd))
+ else:
+ padd_1 = int(padd / 2)
+ padd_2 = padd - padd_1
+
+ if H > W:
+ pad_param = ((0,0),(padd_1,padd_2),(0,0))
+ else:
+ pad_param = ((padd_1,padd_2),(0,0),(0,0))
+
+ image = np.pad(image, pad_param, 'constant', constant_values=pad_value)
+ return image
+
+ salient_object_mask = infer_matting(subject_image)[..., 0]
+ x1, y1, x2, y2 = get_bbox_from_mask(salient_object_mask)
+ subject_image = np.array(subject_image)
+ salient_object_mask[salient_object_mask > 128] = 255
+ salient_object_mask[salient_object_mask < 128] = 0
+ sample_mask = np.concatenate([salient_object_mask[..., None]]*3, axis=2)
+ obj_image = sample_mask / 255 * subject_image + (1 - sample_mask / 255) * 255
+ crop_obj_image = obj_image[y1:y2, x1:x2]
+ crop_pad_obj_image = pad_to_square(crop_obj_image, 255)
+ subject_image = Image.fromarray(crop_pad_obj_image.astype(np.uint8))
+ return subject_image
+
+
+def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
+ if randomize_seed:
+ seed = random.randint(0, MAX_SEED)
+ return seed
+
+def get_example():
+ case = [
+ [
+ "./assets/girl.jpg",
+ "A girl is playing a guitar in street",
+ 0.9,
+ 'Makoto Shinkai style',
+ ],
+ [
+ "./assets/boy.jpg",
+ "A boy is riding a bike in snow",
+ 0.9,
+ 'Makoto Shinkai style',
+ ],
+ ]
+ return case
+
+def run_for_examples(source_image, prompt, scale, style_mode):
+
+ return create_image(
+ input_image=source_image,
+ prompt=prompt,
+ scale=scale,
+ guidance_scale=3.5,
+ num_inference_steps=28,
+ seed=123456,
+ style_mode=style_mode,
+ )
+
+def create_image(input_image,
+ prompt,
+ scale,
+ guidance_scale,
+ num_inference_steps,
+ seed,
+ style_mode=None):
+
+ input_image = remove_bkg(input_image)
+
+ if style_mode is None:
+ images = pipe(
+ prompt=prompt,
+ num_inference_steps=num_inference_steps,
+ guidance_scale=guidance_scale,
+ width=1024,
+ height=1024,
+ subject_image=input_image,
+ subject_scale=scale,
+ generator=torch.manual_seed(seed),
+ ).images
+ else:
+ if style_mode == 'Makoto Shinkai style':
+ lora_file_path = makoto_style_lora_path
+ trigger = 'Makoto Shinkai style'
+ elif style_mode == 'Ghibli style':
+ lora_file_path = ghibli_style_lora_path
+ trigger = 'ghibli style'
+
+ images = pipe.with_style_lora(
+ lora_file_path=lora_file_path,
+ trigger=trigger,
+ prompt=prompt,
+ num_inference_steps=num_inference_steps,
+ guidance_scale=guidance_scale,
+ width=1024,
+ height=1024,
+ subject_image=input_image,
+ subject_scale=scale,
+ generator=torch.manual_seed(seed),
+ ).images
+
+
+ return images
+
+# Description
+title = r"""
+