nodes/aigc-apps-EasyAnimate
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update Prompt && Update Readme && Fix control bug (#131)

Browse Source 
* Update Readme

* Update prompt tips

* Fix bug in ui

* Fix control bug and Update readme

* Update prompt again
This commit is contained in:
Bubbliiiing
2024-11-11 13:48:53 +08:00
committed by GitHub
parent 44539fe77b
commit 9da6b79ece
13 changed files with 445 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+36 -6
View File
@@ -131,8 +131,7 @@ The results displayed are all based on image.
### EasyAnimateV5-12b-zh-InP
Resolution-1024
#### I2V
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
@@ -151,8 +150,6 @@ Resolution-1024
</table>
Resolution-768
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
@@ -170,8 +167,6 @@ Resolution-768
</tr>
</table>
Resolution-512
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
@@ -189,6 +184,41 @@ Resolution-512
</tr>
</table>
#### T2V
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
<video src="https://github.com/user-attachments/assets/eccb0797-4feb-48e9-91d3-5769ce30142b" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/76b3db64-9c7a-4d38-8854-dba940240ceb" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/0b8fab66-8de7-44ff-bd43-8f701bad6bb7" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/9fbddf5f-7fcd-4cc6-9d7c-3bdf1d4ce59e" width="100%" controls autoplay loop></video>
</td>
</tr>
</table>
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
<video src="https://github.com/user-attachments/assets/19c1742b-e417-45ac-97d6-8bf3a80d8e13" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/641e56c8-a3d9-489d-a3a6-42c50a9aeca1" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/2b16be76-518b-44c6-a69b-5c49d76df365" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/e7d9c0fc-136f-405c-9fab-629389e196be" width="100%" controls autoplay loop></video>
</td>
</tr>
</table>
### EasyAnimateV5-12b-zh-Control
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
README_zh-CN.md
+36 -6
View File
@@ -129,8 +129,7 @@ EasyAnimateV5:
### EasyAnimateV5-12b-zh-InP
Resolution-1024
#### I2V
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
@@ -149,8 +148,6 @@ Resolution-1024
</table>
Resolution-768
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
@@ -168,8 +165,6 @@ Resolution-768
</tr>
</table>
Resolution-512
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
@@ -187,6 +182,41 @@ Resolution-512
</tr>
</table>
#### T2V
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
<video src="https://github.com/user-attachments/assets/eccb0797-4feb-48e9-91d3-5769ce30142b" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/76b3db64-9c7a-4d38-8854-dba940240ceb" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/0b8fab66-8de7-44ff-bd43-8f701bad6bb7" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/9fbddf5f-7fcd-4cc6-9d7c-3bdf1d4ce59e" width="100%" controls autoplay loop></video>
</td>
</tr>
</table>
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
<tr>
<td>
<video src="https://github.com/user-attachments/assets/19c1742b-e417-45ac-97d6-8bf3a80d8e13" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/641e56c8-a3d9-489d-a3a6-42c50a9aeca1" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/2b16be76-518b-44c6-a69b-5c49d76df365" width="100%" controls autoplay loop></video>
</td>
<td>
<video src="https://github.com/user-attachments/assets/e7d9c0fc-136f-405c-9fab-629389e196be" width="100%" controls autoplay loop></video>
</td>
</tr>
</table>
### EasyAnimateV5-12b-zh-Control
<table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
comfyui/comfyui_nodes.py
+10 -6
View File
@@ -265,12 +265,16 @@ class LoadEasyAnimateModel:
)
else:
pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Control.from_pretrained(
model_name,
vae=vae,
transformer=transformer,
scheduler=scheduler,
torch_dtype=weight_dtype
)
model_name,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=vae,
transformer=transformer,
scheduler=scheduler,
torch_dtype=weight_dtype
)
if GPU_memory_mode == "sequential_cpu_offload":
pipeline.enable_sequential_cpu_offload()
elif GPU_memory_mode == "model_cpu_offload_and_qfloat8":
comfyui/v5/easyanimatev5_workflow_i2v.json
+64 -38
View File
@@ -1,5 +1,5 @@
{
"last_node_id": 83,
"last_node_id": 84,
"last_link_id": 48,
"nodes": [
{
@@ -52,31 +52,6 @@
"color": "#432",
"bgcolor": "#653"
},
{
"id": 78,
"type": "Note",
"pos": {
"0": 18,
"1": -46
},
"size": {
"0": 210,
"1": 58
},
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"You can write prompt here\n（你可以在此填写提示词）"
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 17,
"type": "VHS_VideoCombine",
@@ -86,10 +61,10 @@
},
"size": [
390.9534912109375,
535.9734235491071
310
],
"flags": {},
"order": 8,
"order": 9,
"mode": 0,
"inputs": [
{
@@ -97,7 +72,8 @@
"type": "IMAGE",
"link": 42,
"slot_index": 0,
"label": "图像"
"label": "图像",
"shape": 7
},
{
"name": "audio",
@@ -168,7 +144,7 @@
"1": 282
},
"flags": {},
"order": 7,
"order": 8,
"mode": 0,
"inputs": [
{
@@ -235,7 +211,7 @@
"1": 154
},
"flags": {},
"order": 3,
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [
@@ -272,7 +248,7 @@
"1": 314
},
"flags": {},
"order": 4,
"order": 3,
"mode": 0,
"inputs": [],
"outputs": [
@@ -315,7 +291,7 @@
"1": 156.71620178222656
},
"flags": {},
"order": 5,
"order": 4,
"mode": 0,
"inputs": [],
"outputs": [
@@ -349,7 +325,7 @@
"1": 183.83506774902344
},
"flags": {},
"order": 6,
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [
@@ -368,8 +344,58 @@
"Node name for S&R": "TextBox"
},
"widgets_values": [
"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
"扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
]
},
{
"id": 78,
"type": "Note",
"pos": {
"0": 18,
"1": -46
},
"size": {
"0": 210,
"1": 58
},
"flags": {},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"You can write prompt here\n（你可以在此填写提示词）"
],
"color": "#432",
"bgcolor": "#653"
},
{
"id": 84,
"type": "Note",
"pos": {
"0": -98,
"1": 198
},
"size": [
326.1556114026207,
145.20905264909447
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)"
],
"color": "#432",
"bgcolor": "#653"
}
],
"links": [
@@ -455,10 +481,10 @@
"config": {},
"extra": {
"ds": {
"scale": 0.7247295000000005,
"scale": 0.7513148009015777,
"offset": [
395.9831007505868,
450.6554192121478
206.91128312862938,
440.0942364134056
]
},
"workspace_info": {
comfyui/v5/easyanimatev5_workflow_t2v.json
+47 -21
View File
@@ -1,5 +1,5 @@
{
"last_node_id": 88,
"last_node_id": 89,
"last_link_id": 53,
"nodes": [
{
@@ -101,7 +101,7 @@
"1": 290
},
"flags": {},
"order": 5,
"order": 6,
"mode": 0,
"inputs": [
{
@@ -214,7 +214,7 @@
"Node name for S&R": "TextBox"
},
"widgets_values": [
"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
"扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
]
},
{
@@ -226,10 +226,10 @@
},
"size": [
390.9534912109375,
535.9734235491071
310
],
"flags": {},
"order": 6,
"order": 7,
"mode": 0,
"inputs": [
{
@@ -237,7 +237,8 @@
"type": "IMAGE",
"link": 50,
"slot_index": 0,
"label": "图像"
"label": "图像",
"shape": 7
},
{
"name": "audio",
@@ -295,6 +296,31 @@
}
}
}
},
{
"id": 89,
"type": "Note",
"pos": {
"0": -97,
"1": 193
},
"size": [
326.1556114026207,
145.20905264909447
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)"
],
"color": "#432",
"bgcolor": "#653"
}
],
"links": [
@@ -332,18 +358,6 @@
]
],
"groups": [
{
"title": "Prompts",
"bounding": [
218,
-127,
450,
483
],
"color": "#3f789e",
"font_size": 24,
"flags": {}
},
{
"title": "Load EasyAnimate",
"bounding": [
@@ -355,15 +369,27 @@
"color": "#b06634",
"font_size": 24,
"flags": {}
},
{
"title": "Prompts",
"bounding": [
218,
-127,
450,
483
],
"color": "#3f789e",
"font_size": 24,
"flags": {}
}
],
"config": {},
"extra": {
"ds": {
"scale": 0.8769226950000006,
"scale": 0.7513148009015777,
"offset": [
89.91314837012095,
505.1888927984372
206.91128312862938,
440.0942364134056
]
},
"workspace_info": {
comfyui/v5/easyanimatev5_workflow_v2v.json
+36 -10
View File
@@ -1,5 +1,5 @@
{
"last_node_id": 87,
"last_node_id": 88,
"last_link_id": 52,
"nodes": [
{
@@ -145,7 +145,7 @@
"Node name for S&R": "TextBox"
},
"widgets_values": [
"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
"扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
]
},
{
@@ -194,7 +194,7 @@
"1": 306
},
"flags": {},
"order": 7,
"order": 8,
"mode": 0,
"inputs": [
{
@@ -262,10 +262,10 @@
},
"size": [
390.9534912109375,
535.9734235491071
310
],
"flags": {},
"order": 8,
"order": 9,
"mode": 0,
"inputs": [
{
@@ -273,7 +273,8 @@
"type": "IMAGE",
"link": 48,
"slot_index": 0,
"label": "图像"
"label": "图像",
"shape": 7
},
{
"name": "audio",
@@ -341,7 +342,7 @@
},
"size": [
252.056640625,
408.6037946428571
262
],
"flags": {},
"order": 6,
@@ -416,6 +417,31 @@
}
}
}
},
{
"id": 88,
"type": "Note",
"pos": {
"0": -97,
"1": 195
},
"size": [
326.1556091308594,
145.20904541015625
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)"
],
"color": "#432",
"bgcolor": "#653"
}
],
"links": [
@@ -501,10 +527,10 @@
"config": {},
"extra": {
"ds": {
"scale": 0.6588450000000011,
"scale": 0.7513148009015777,
"offset": [
599.4920942053536,
496.29399099271234
206.91128312862938,
440.0942364134056
]
},
"workspace_info": {
comfyui/v5/easyanimatev5_workflow_v2v_control.json
+36 -10
View File
@@ -1,5 +1,5 @@
{
"last_node_id": 87,
"last_node_id": 88,
"last_link_id": 53,
"nodes": [
{
@@ -108,7 +108,7 @@
"Node name for S&R": "TextBox"
},
"widgets_values": [
"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
"扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
]
},
{
@@ -120,10 +120,10 @@
},
"size": [
390.9534912109375,
973.1686096191406
310
],
"flags": {},
"order": 8,
"order": 9,
"mode": 0,
"inputs": [
{
@@ -131,7 +131,8 @@
"type": "IMAGE",
"link": 48,
"slot_index": 0,
"label": "图像"
"label": "图像",
"shape": 7
},
{
"name": "audio",
@@ -236,7 +237,7 @@
},
"size": [
252.056640625,
688.5451388888889
262
],
"flags": {},
"order": 5,
@@ -358,7 +359,7 @@
"1": 306
},
"flags": {},
"order": 7,
"order": 8,
"mode": 0,
"inputs": [
{
@@ -416,6 +417,31 @@
1,
"DDIM"
]
},
{
"id": 88,
"type": "Note",
"pos": {
"0": -99,
"1": 197
},
"size": [
326.1556114026207,
145.20905264909447
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [],
"outputs": [],
"properties": {
"text": ""
},
"widgets_values": [
"Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)"
],
"color": "#432",
"bgcolor": "#653"
}
],
"links": [
@@ -501,10 +527,10 @@
"config": {},
"extra": {
"ds": {
"scale": 0.7247295000000012,
"scale": 0.7513148009015777,
"offset": [
302.3321359435954,
322.8143100800612
206.91128312862938,
440.0942364134056
]
},
"workspace_info": {
easyanimate/api/api.py
+1 -1
View File
@@ -93,7 +93,7 @@ def infer_forward_api(_: gr.Blocks, app: FastAPI, controller):
lora_model_path = datas.get('lora_model_path', 'none')
lora_alpha_slider = datas.get('lora_alpha_slider', 0.55)
prompt_textbox = datas.get('prompt_textbox', None)
negative_prompt_textbox = datas.get('negative_prompt_textbox', 'Blurring, mutation, deformation, distortion, dark and solid, comics.')
negative_prompt_textbox = datas.get('negative_prompt_textbox', 'Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art.')
sampler_dropdown = datas.get('sampler_dropdown', 'Euler')
sample_step_slider = datas.get('sample_step_slider', 30)
resize_method = datas.get('resize_method', "Generate by")
easyanimate/ui/ui.py
+146 -100
View File
@@ -39,6 +39,8 @@ from easyanimate.pipeline.pipeline_easyanimate_multi_text_encoder import \
EasyAnimatePipeline_Multi_Text_Encoder
from easyanimate.pipeline.pipeline_easyanimate_multi_text_encoder_inpaint import \
EasyAnimatePipeline_Multi_Text_Encoder_Inpaint
from easyanimate.pipeline.pipeline_easyanimate_multi_text_encoder_control import \
EasyAnimatePipeline_Multi_Text_Encoder_Control
from easyanimate.utils.lora_utils import merge_lora, unmerge_lora
from easyanimate.utils.utils import (
get_image_to_video_latent, get_video_to_video_latent,
@@ -225,56 +227,69 @@ class EasyAnimateController:
subfolder="scheduler"
)
if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False):
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Inpaint.from_pretrained(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
if self.model_type == "Inpaint":
if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False):
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Inpaint.from_pretrained(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
else:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder.from_pretrained(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
else:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder.from_pretrained(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimateInpaintPipeline(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
else:
self.pipeline = EasyAnimatePipeline(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
else:
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimateInpaintPipeline(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
else:
self.pipeline = EasyAnimatePipeline(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Control.from_pretrained(
diffusion_transformer_dropdown,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
if self.GPU_memory_mode == "sequential_cpu_offload":
self.pipeline.enable_sequential_cpu_offload()
@@ -283,7 +298,7 @@ class EasyAnimateController:
self.pipeline.enable_autocast_float8_transformer()
convert_weight_dtype_wrapper(self.pipeline.transformer, self.weight_dtype)
else:
self.GPU_memory_mode.enable_model_cpu_offload()
self.pipeline.enable_model_cpu_offload()
print("Update diffusion transformer done")
return gr.update()
@@ -752,7 +767,13 @@ def ui(GPU_memory_mode, weight_dtype):
)
prompt_textbox = gr.Textbox(label="Prompt (正向提示词)", lines=2, value="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.")
negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="Blurring, mutation, deformation, distortion, dark and solid, comics." )
gr.Markdown(
"""
Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability. Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性。在neg prompt中添加"安静，固定"等词语可以增加动态性。
"""
)
negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code.. " )
with gr.Row():
with gr.Column():
@@ -1058,56 +1079,69 @@ class EasyAnimateController_Modelscope:
subfolder="scheduler"
)
if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False):
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Inpaint.from_pretrained(
model_name,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
if model_type == "Inpaint":
if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False):
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Inpaint.from_pretrained(
model_name,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
else:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder.from_pretrained(
model_name,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
else:
self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder.from_pretrained(
model_name,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimateInpaintPipeline(
model_name,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
else:
self.pipeline = EasyAnimatePipeline(
model_name,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
else:
if self.transformer.config.in_channels != self.vae.config.latent_channels:
self.pipeline = EasyAnimateInpaintPipeline(
model_name,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype,
clip_image_encoder=clip_image_encoder,
clip_image_processor=clip_image_processor,
)
else:
self.pipeline = EasyAnimatePipeline(
model_name,
text_encoder=text_encoder,
tokenizer=tokenizer,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=self.weight_dtype
)
pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Control.from_pretrained(
model_name,
text_encoder=text_encoder,
text_encoder_2=text_encoder_2,
tokenizer=tokenizer,
tokenizer_2=tokenizer_2,
vae=self.vae,
transformer=self.transformer,
scheduler=scheduler,
torch_dtype=weight_dtype
)
if GPU_memory_mode == "sequential_cpu_offload":
self.pipeline.enable_sequential_cpu_offload()
@@ -1407,7 +1441,13 @@ def ui_modelscope(model_type, edition, config_path, model_name, savedir_sample,
)
prompt_textbox = gr.Textbox(label="Prompt (正向提示词)", lines=2, value="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.")
negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="Blurring, mutation, deformation, distortion, dark and solid, comics." )
gr.Markdown(
"""
Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability. Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性。在neg prompt中添加"安静，固定"等词语可以增加动态性。
"""
)
negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code.. " )
with gr.Row():
with gr.Column():
@@ -1821,7 +1861,13 @@ def ui_eas(edition, config_path, model_name, savedir_sample):
)
prompt_textbox = gr.Textbox(label="Prompt", lines=2, value="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.")
negative_prompt_textbox = gr.Textbox(label="Negative prompt", lines=2, value="Blurring, mutation, deformation, distortion, dark and solid, comics." )
gr.Markdown(
"""
Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability. Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性。在neg prompt中添加"安静，固定"等词语可以增加动态性。
"""
)
negative_prompt_textbox = gr.Textbox(label="Negative prompt", lines=2, value="Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code.. " )
with gr.Row():
with gr.Column():
predict_i2v.py
+7 -2
View File
@@ -70,10 +70,15 @@ validation_image_end = None
# EasyAnimateV1, V2 and V3 support English.
# EasyAnimateV4 and V5 support English and Chinese.
# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性
# 在neg prompt中添加"安静，固定"等词语可以增加动态性。
prompt = "一条狗正在摇头。质量高、杰作、最佳品质、高分辨率、超精细、梦幻般。"
negative_prompt = "模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
negative_prompt = "扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
#
# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability
# Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
# prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic."
# negative_prompt = "Blurring, mutation, deformation, distortion, dark and solid, comics."
# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code."
guidance_scale = 6.0
seed = 43
num_inference_steps = 50
predict_t2v.py
+10 -4
View File
@@ -63,12 +63,18 @@ fps = 8
# Use torch.float16 if GPU does not support torch.bfloat16
# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16
weight_dtype = torch.bfloat16
# # EasyAnimateV1, V2 and V3 support English.
# EasyAnimateV1, V2 and V3 support English.
# EasyAnimateV4 and V5 support English and Chinese.
# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性
# 在neg prompt中添加"安静，固定"等词语可以增加动态性。
prompt = "一条狗正在摇头。质量高、杰作、最佳品质、高分辨率、超精细、梦幻般。"
negative_prompt = "模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
# prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic."
# negative_prompt = "Blurring, mutation, deformation, distortion, dark and solid, comics."
negative_prompt = "扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
#
# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability
# Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
# prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic."
# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code."
guidance_scale = 6.0
seed = 43
num_inference_steps = 50
predict_v2v.py
+8 -4
View File
@@ -65,13 +65,17 @@ weight_dtype = torch.bfloat16
validation_video = "asset/1.mp4"
denoise_strength = 0.70
# prompts
# # EasyAnimateV1, V2 and V3 support English.
# EasyAnimateV1, V2 and V3 support English.
# EasyAnimateV4 and V5 support English and Chinese.
# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性
# 在neg prompt中添加"安静，固定"等词语可以增加动态性。
prompt = "一只猫正在弹吉他。"
negative_prompt = "模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
negative_prompt = "扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
#
# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability
# Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
# prompt = "A cute cat is playing the guitar. "
# negative_prompt = "Blurring, mutation, deformation, distortion, dark and solid, comics."
# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code.. "
guidance_scale = 6.0
seed = 43
num_inference_steps = 50
predict_v2v_control.py
+8 -1
View File
@@ -59,8 +59,15 @@ control_video = "asset/pose.mp4"
# EasyAnimateV1, V2 and V3 support English.
# EasyAnimateV4 and V5 support English and Chinese.
# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性
# 在neg prompt中添加"安静，固定"等词语可以增加动态性。
prompt = "一位年轻女子，有着美丽清澈的眼睛和金发，穿着白色的衣服在扭动身体，相机聚焦在她的脸上。质量高、杰作、最佳品质、高分辨率、超精细、梦幻般。"
negative_prompt = "模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。"
negative_prompt = "扭曲的身体，肢体残缺，文本字幕，漫画，静止，丑陋，错误，乱码。"
#
# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability
# Adding words such as "quiet, solid" to the neg prompt can increase dynamism.
# prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic."
# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code."
guidance_scale = 6.0
seed = 43
num_inference_steps = 50