Compare commits
49
Commits
v1.0.0_dev
...
v1.2.9_dev
| Author | SHA1 | Date | |
|---|---|---|---|
 皓童
|
af5cf1ccc0 | ||
|
皓童
|
0ed54e05e4 | ||
|
皓童
|
27a2b7833f | ||
|
皓童
|
6c8ce55888 | ||
|
皓童
|
448cdba522 | ||
|
皓童
|
2a29446d45 | ||
|
maochaojie
|
cd33b4ab15 | ||
|
maochaojie
|
7d7943fed3 | ||
|
jiangzeyinzi
|
d48b2f110f | ||
|
Great
|
82486adf38 | ||
|
maochaojie
|
a683061c6f | ||
|
mcj
|
4ec0492897 | ||
|
LouieStark
|
aac85fa94f | ||
|
LouieStark
|
3f267aaea2 | ||
|
LouieStark
|
53357f95d6 | ||
|
mcj
|
02c0ba9757 | ||
|
LouieStark
|
cef93bdbfe | ||
|
jiangzeyinzi
|
82132ff3a1 | ||
|
LouieStark
|
b886400e06 | ||
|
mcj
|
73984c4f9e | ||
|
LouieStark
|
f98adabeb3 | ||
|
jiangzeyinzi
|
edb46a615c | ||
|
LouieStark
|
fe3e11b49e | ||
|
LouieStark
|
e6b43f19f6 | ||
|
jiangzeyinzi
|
d9b207cf5b | ||
|
LouieStark
|
986349deac | ||
|
LouieStark
|
3f047be43c | ||
|
LouieStark
|
e8d8e63cba | ||
|
jiangzeyinzi
|
eac03e9856 | ||
|
jiangzeyinzi
|
379b94ab4f | ||
|
jiangzeyinzi
|
342c6b8a15 | ||
|
jiangzeyinzi
|
4ddcb08c7b | ||
|
jiangzeyinzi
|
b2169a1597 | ||
|
jiangzeyinzi
|
cffd54a02a | ||
|
zeyinzi.jzyz
|
0bba2c319d | ||
|
LouieStark
|
7d6451efad | ||
|
LouieStark
|
a5decd17aa | ||
|
jiangzeyinzi
|
8a14866562 | ||
|
jiangzeyinzi
|
5a94f6c4a2 | ||
|
Great
|
4ad0f6038c | ||
|
Great
|
4f56623627 | ||
|
jiangzeyinzi
|
30afc0a1de | ||
|
Great
|
aeabef75b4 | ||
|
jiangzeyinzi
|
9376149415 | ||
|
zeyinzi.jzyz
|
01fd8335af | ||
|
Zhen Han
|
7a9f90efb2 | ||
|
Zhen Han
|
cbbef4a2da | ||
|
hanzhen.hz
|
b242449b25 | ||
|
jiangzeyinzi
|
2fb8fd1872 |
+2
-2
@@ -9,12 +9,12 @@
|
|||||||
*.bin
|
*.bin
|
||||||
*.idea
|
*.idea
|
||||||
*.csv
|
*.csv
|
||||||
|
cache
|
||||||
build
|
build
|
||||||
dist
|
dist
|
||||||
dev
|
dev
|
||||||
scepter.egg-info
|
scepter.egg-info
|
||||||
.readthedocs.yml
|
.readthedocs.yml
|
||||||
1.9
|
|
||||||
#MANIFEST.in
|
|
||||||
*resources
|
*resources
|
||||||
*.ipynb_checkpoints*
|
*.ipynb_checkpoints*
|
||||||
|
*.vscode
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 74 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 140 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 99 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 97 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 48 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 97 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 378 KiB |
@@ -0,0 +1,163 @@
|
|||||||
|
{
|
||||||
|
"last_node_id": 4,
|
||||||
|
"last_link_id": 3,
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "PreviewImage",
|
||||||
|
"pos": {
|
||||||
|
"0": 937,
|
||||||
|
"1": 243
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 284.03680419921875,
|
||||||
|
"1": 246
|
||||||
|
},
|
||||||
|
"flags": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"order": 2,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "images",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "PreviewImage"
|
||||||
|
},
|
||||||
|
"widgets_values": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "ParameterNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 0,
|
||||||
|
"1": 244
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 311.9849853515625,
|
||||||
|
"1": 236.4765625
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 0,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
3
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ParameterNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"ddim",
|
||||||
|
50,
|
||||||
|
5,
|
||||||
|
0.5,
|
||||||
|
"trailing",
|
||||||
|
1024,
|
||||||
|
1024,
|
||||||
|
2024
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "ModelNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 430,
|
||||||
|
"1": 243
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 400,
|
||||||
|
"1": 234
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 1,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "parameters",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 3,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mantras",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tuners",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "controls",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "IMAGE",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"slot_index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ModelNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0",
|
||||||
|
"ModelScope",
|
||||||
|
"a cat",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"links": [
|
||||||
|
[
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
2,
|
||||||
|
0,
|
||||||
|
"IMAGE"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
3,
|
||||||
|
4,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
"CONDITIONING"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"groups": [],
|
||||||
|
"config": {},
|
||||||
|
"extra": {
|
||||||
|
"ds": {
|
||||||
|
"scale": 0.9229599817706423,
|
||||||
|
"offset": [
|
||||||
|
128.91466506592707,
|
||||||
|
43.07074577975898
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"version": 0.4
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 291 KiB |
@@ -0,0 +1,202 @@
|
|||||||
|
{
|
||||||
|
"last_node_id": 5,
|
||||||
|
"last_link_id": 4,
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "PreviewImage",
|
||||||
|
"pos": {
|
||||||
|
"0": 937,
|
||||||
|
"1": 243
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 284.03680419921875,
|
||||||
|
"1": 246
|
||||||
|
},
|
||||||
|
"flags": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"order": 3,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "images",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "PreviewImage"
|
||||||
|
},
|
||||||
|
"widgets_values": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "ParameterNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 0,
|
||||||
|
"1": 244
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 311.9849853515625,
|
||||||
|
"1": 236.4765625
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 0,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
3
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ParameterNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"ddim",
|
||||||
|
50,
|
||||||
|
5,
|
||||||
|
0.5,
|
||||||
|
"trailing",
|
||||||
|
1024,
|
||||||
|
1024,
|
||||||
|
2024
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "ModelNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 430,
|
||||||
|
"1": 243
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 400,
|
||||||
|
"1": 234
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 2,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "parameters",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 3,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mantras",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 4,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tuners",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "controls",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "IMAGE",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"slot_index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ModelNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0",
|
||||||
|
"ModelScope",
|
||||||
|
"a cat",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "MantrasNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 3,
|
||||||
|
"1": 547
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 315,
|
||||||
|
"1": 58
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 1,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
4
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "MantrasNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"Flat 2D Art"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"links": [
|
||||||
|
[
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
2,
|
||||||
|
0,
|
||||||
|
"IMAGE"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
3,
|
||||||
|
4,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
4,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
"CONDITIONING"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"groups": [],
|
||||||
|
"config": {},
|
||||||
|
"extra": {
|
||||||
|
"ds": {
|
||||||
|
"scale": 1.0152559799477068,
|
||||||
|
"offset": [
|
||||||
|
58.35814123566077,
|
||||||
|
-65.12749162217047
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"version": 0.4
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 230 KiB |
@@ -0,0 +1,242 @@
|
|||||||
|
{
|
||||||
|
"last_node_id": 6,
|
||||||
|
"last_link_id": 5,
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "PreviewImage",
|
||||||
|
"pos": {
|
||||||
|
"0": 937,
|
||||||
|
"1": 243
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 284.03680419921875,
|
||||||
|
"1": 246
|
||||||
|
},
|
||||||
|
"flags": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"order": 4,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "images",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "PreviewImage"
|
||||||
|
},
|
||||||
|
"widgets_values": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "ParameterNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 0,
|
||||||
|
"1": 244
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 311.9849853515625,
|
||||||
|
"1": 236.4765625
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 0,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
3
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ParameterNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"ddim",
|
||||||
|
50,
|
||||||
|
5,
|
||||||
|
0.5,
|
||||||
|
"trailing",
|
||||||
|
1024,
|
||||||
|
1024,
|
||||||
|
2024
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "MantrasNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 3,
|
||||||
|
"1": 547
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 315,
|
||||||
|
"1": 58
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 1,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
4
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "MantrasNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"Flat 2D Art"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "ModelNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 430,
|
||||||
|
"1": 243
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 400,
|
||||||
|
"1": 234
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 3,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "parameters",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 3,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mantras",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 4,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tuners",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 5,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "controls",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": null,
|
||||||
|
"shape": 7
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "IMAGE",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"slot_index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ModelNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0",
|
||||||
|
"ModelScope",
|
||||||
|
"a cat",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"type": "TunerNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 8,
|
||||||
|
"1": 680
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 315,
|
||||||
|
"1": 82
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 2,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
5
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "TunerNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0_Pencil Sketch Drawing",
|
||||||
|
1
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"links": [
|
||||||
|
[
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
2,
|
||||||
|
0,
|
||||||
|
"IMAGE"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
3,
|
||||||
|
4,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
4,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
5,
|
||||||
|
6,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
"CONDITIONING"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"groups": [],
|
||||||
|
"config": {},
|
||||||
|
"extra": {
|
||||||
|
"ds": {
|
||||||
|
"scale": 0.6934334949441366,
|
||||||
|
"offset": [
|
||||||
|
450.0923948318967,
|
||||||
|
30.783653239467935
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"version": 0.4
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 355 KiB |
@@ -0,0 +1,366 @@
|
|||||||
|
{
|
||||||
|
"last_node_id": 11,
|
||||||
|
"last_link_id": 9,
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "ModelNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 439,
|
||||||
|
"1": 123
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 400,
|
||||||
|
"1": 234
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 6,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "parameters",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 3,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "mantras",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 4,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "tuners",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 5,
|
||||||
|
"shape": 7
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "controls",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"link": 6,
|
||||||
|
"shape": 7
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "IMAGE",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"slot_index": 0
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ModelNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0",
|
||||||
|
"ModelScope",
|
||||||
|
"a cat",
|
||||||
|
""
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "MantrasNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 13,
|
||||||
|
"1": 336
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 315,
|
||||||
|
"1": 58
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 0,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
4
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "MantrasNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"Flat 2D Art"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"type": "TunerNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 12,
|
||||||
|
"1": 455
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 315,
|
||||||
|
"1": 82
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 1,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
5
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "TunerNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0_Pencil Sketch Drawing",
|
||||||
|
1
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"type": "NoteNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 845,
|
||||||
|
"1": 506
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 398.3059387207031,
|
||||||
|
"1": 210.83267211914062
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 2,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "NoteNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"This is a sample for quickly setting up ComfyUI using the Scepter open-source library:\n\n1) The first run involves downloading the models. By default, we will automatically pull models from ModelScope. The initial download may take some time, and you can also adjust the model source to change the model address.\n\n2) Currently, it supports various base models, basic settings for some inference hyperparameters, mantra settings, tuning model settings, and conditional generation node.\n\n3) In the future, we will gradually integrate interesting features to enrich the use cases.\n\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"type": "LoadImage",
|
||||||
|
"pos": {
|
||||||
|
"0": 471,
|
||||||
|
"1": 451
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 315,
|
||||||
|
"1": 314
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 3,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "IMAGE",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [
|
||||||
|
7
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "MASK",
|
||||||
|
"type": "MASK",
|
||||||
|
"links": null
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "LoadImage"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"cat.jpg",
|
||||||
|
"image"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"type": "ControlNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 15,
|
||||||
|
"1": 600
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 330,
|
||||||
|
"1": 198
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 5,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "source_image",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 7
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
6
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Control Image",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"links": [],
|
||||||
|
"slot_index": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ControlNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"SD_XL1.0_color",
|
||||||
|
"Color",
|
||||||
|
"CenterCrop",
|
||||||
|
1,
|
||||||
|
1024,
|
||||||
|
1024
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "PreviewImage",
|
||||||
|
"pos": {
|
||||||
|
"0": 959,
|
||||||
|
"1": 121
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 284.03680419921875,
|
||||||
|
"1": 246
|
||||||
|
},
|
||||||
|
"flags": {
|
||||||
|
"collapsed": false
|
||||||
|
},
|
||||||
|
"order": 7,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [
|
||||||
|
{
|
||||||
|
"name": "images",
|
||||||
|
"type": "IMAGE",
|
||||||
|
"link": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"outputs": [],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "PreviewImage"
|
||||||
|
},
|
||||||
|
"widgets_values": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "ParameterNode",
|
||||||
|
"pos": {
|
||||||
|
"0": 13,
|
||||||
|
"1": 43
|
||||||
|
},
|
||||||
|
"size": {
|
||||||
|
"0": 311.9849853515625,
|
||||||
|
"1": 236.4765625
|
||||||
|
},
|
||||||
|
"flags": {},
|
||||||
|
"order": 4,
|
||||||
|
"mode": 0,
|
||||||
|
"inputs": [],
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "Result",
|
||||||
|
"type": "CONDITIONING",
|
||||||
|
"links": [
|
||||||
|
3
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"Node name for S&R": "ParameterNode"
|
||||||
|
},
|
||||||
|
"widgets_values": [
|
||||||
|
"ddim",
|
||||||
|
50,
|
||||||
|
5,
|
||||||
|
0.5,
|
||||||
|
"trailing",
|
||||||
|
1024,
|
||||||
|
1024,
|
||||||
|
2024
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"links": [
|
||||||
|
[
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
2,
|
||||||
|
0,
|
||||||
|
"IMAGE"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
3,
|
||||||
|
4,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
4,
|
||||||
|
5,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
5,
|
||||||
|
6,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
2,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
6,
|
||||||
|
7,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
3,
|
||||||
|
"CONDITIONING"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
7,
|
||||||
|
8,
|
||||||
|
0,
|
||||||
|
7,
|
||||||
|
0,
|
||||||
|
"IMAGE"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"groups": [],
|
||||||
|
"config": {},
|
||||||
|
"extra": {
|
||||||
|
"ds": {
|
||||||
|
"scale": 0.7627768444385667,
|
||||||
|
"offset": [
|
||||||
|
254.62860058494985,
|
||||||
|
87.05734194144193
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"version": 0.4
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 634 KiB |
@@ -56,7 +56,7 @@ Given an original image, image editing aims to generate an image that align with
|
|||||||
- More models will be released in the future.
|
- More models will be released in the future.
|
||||||
|
|
||||||
## Run StyleBooth
|
## Run StyleBooth
|
||||||
- Code implementation: See model configuration and code based on [🪄SCEPTER](https://github.com/modelscope/scepter).
|
- Code implementation: See model configuration and code based on [🪄SCEPTER](https://github.com/modelscope/scepter/blob/main/docs/en/tasks/stylebooth.md).
|
||||||
|
|
||||||
- Demo: Try [🖥️SCEPTER Studio](https://github.com/modelscope/scepter/tree/main?tab=readme-ov-file#%EF%B8%8F-scepter-studio).
|
- Demo: Try [🖥️SCEPTER Studio](https://github.com/modelscope/scepter/tree/main?tab=readme-ov-file#%EF%B8%8F-scepter-studio).
|
||||||
|
|
||||||
@@ -114,3 +114,17 @@ class DiffusionInferenceTest(unittest.TestCase):
|
|||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## StyleTuner and De-StyleTuner.
|
||||||
|
|
||||||
|
### Base I2I Model.
|
||||||
|
|
||||||
|
For style and de-style tuning, we use a private high-resolution I2I model trained with [InstructPix2Pix dataset](https://instruct-pix2pix.eecs.berkeley.edu/) as base model. However, one can try the same tunning process using this [yaml](https://github.com/modelscope/scepter/blob/main/scepter/methods/edit/edit_512_lora.yaml) based on any other I2I model, such as StyleBooth (shown in this yaml), [InstructPix2Pix](https://github.com/timothybrooks/instruct-pix2pix) or [MagicBrush](https://github.com/OSU-NLP-Group/MagicBrush).
|
||||||
|
|
||||||
|
### Training Data.
|
||||||
|
|
||||||
|
Please check the zips for correct format: [De-Text](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fdetext.zip), [Image2Hed](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fhed_pair.zip), [Image2Depth](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fimage2depth.zip), [Depth2Image](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fdepth2image.zip).
|
||||||
|
|
||||||
|
### Launch.
|
||||||
|
|
||||||
|
See [code](https://github.com/modelscope/scepter/blob/cbbef4a2da5b66fc33b9f8ece7f2fb4aac9d6e3c/tests/tools/test_train.py#L220) for more information.
|
||||||
|
|||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if sys.argv[0] == 'install.py':
|
||||||
|
sys.path.append('.') # for portable version
|
||||||
|
|
||||||
|
source_folder = os.path.join(os.path.dirname(__file__), "scepter/workflow")
|
||||||
|
current_dir = os.path.dirname(__file__)
|
||||||
|
destination_folder = os.path.join(os.path.dirname(current_dir), "ComfyUI-Scepter")
|
||||||
|
|
||||||
|
if not os.path.exists(destination_folder):
|
||||||
|
shutil.copytree(source_folder, destination_folder)
|
||||||
|
print(f"{os.path.abspath(source_folder)} copy to {os.path.abspath(destination_folder)} success!")
|
||||||
|
else:
|
||||||
|
print(f"{os.path.abspath(destination_folder)} exist.")
|
||||||
|
|
||||||
|
# pip install scepter
|
||||||
|
subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'scepter'])
|
||||||
@@ -14,10 +14,18 @@ SCEPTER integrates popular community-driven implementations as well as proprieta
|
|||||||
SCEPTER offers 3 core components:
|
SCEPTER offers 3 core components:
|
||||||
- [Generative training and inference framework](#tutorials)
|
- [Generative training and inference framework](#tutorials)
|
||||||
- [Easy implementation of popular approaches](#currently-supported-approaches)
|
- [Easy implementation of popular approaches](#currently-supported-approaches)
|
||||||
- [Interactive user interface: SCEPTER Studio](#launch)
|
- [Interactive user interface: SCEPTER Studio & Comfy UI](#launch)
|
||||||
|
|
||||||
|
|
||||||
## 🎉 News
|
## 🎉 News
|
||||||
|
- [🔥🔥🔥2024.11]: We're excited to announce the upcoming release of the [ACE-0.6b-1024px](https://huggingface.co/scepter-studio/ACE-0.6B-1024px) model,
|
||||||
|
which significantly enhances image generation quality compared with [ACE-0.6b-512px](https://huggingface.co/scepter-studio/ACE-0.6B-512px). The detailed documents can be found at [ACE repo](https://github.com/ali-vilab/ACE.git).
|
||||||
|
At the same time, based on the editing results of ACE, combined with the powerful text-to-image capabilities of the [FLUX-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev) model through SDEdit as an image quality refiner, the quality of image editing can be further enhanced.
|
||||||
|
- [🔥2024.11]: Supports video files, video annotation, caption translation in data management, and inference & training of the [CogVideoX](https://arxiv.org/abs/2408.06072).
|
||||||
|
- [2024.10]: We are pleased to announce the release of the code for [ACE](https://arxiv.org/abs/2410.00086), supporting Customized Training / Comfy UI Workflow / gradio-based ChatBot Interface.
|
||||||
|
- [2024.10]: Support for inference and tuning with [FLUX](https://huggingface.co/black-forest-labs/FLUX.1-dev), as well as for building [ComfyUI](https://github.com/comfyanonymous/ComfyUI) workflows using this framework.
|
||||||
|
- [2024.09]: We introduce **ACE**, an **A**ll-round **C**reator and **E**ditor adept at executing a diverse array of image editing tasks tailored to your specifications. Built upon the cutting-edge Diffusion Transformer architecture, ACE has been extensively trained on a comprehensive dataset to seamlessly interpret and execute any natural language instruction. For further information, please consult the [project page](https://ali-vilab.github.io/ace-page/).
|
||||||
|
- [2024.07]: Support the inference and training of open-source generative models based on the [DiT](https://arxiv.org/abs/2212.09748) architecture, such as [SD3](https://arxiv.org/pdf/2403.03206) and [PixArt](https://arxiv.org/abs/2310.00426).
|
||||||
- [2024.05]: Introducing SCEPTER v1, supporting customized image edit tasks! Simply provide 10 image pairs, SCEPTER will tune an edit tuner for your own Image-to-Image tasks, like `Clay Style`, `De-Text`, `Segmentation`, etc.
|
- [2024.05]: Introducing SCEPTER v1, supporting customized image edit tasks! Simply provide 10 image pairs, SCEPTER will tune an edit tuner for your own Image-to-Image tasks, like `Clay Style`, `De-Text`, `Segmentation`, etc.
|
||||||
- [2024.04]: New [StyleBooth](https://ali-vilab.github.io/stylebooth-page/) demo on SCEPTER Studio for`Text-Based Style Editing`.
|
- [2024.04]: New [StyleBooth](https://ali-vilab.github.io/stylebooth-page/) demo on SCEPTER Studio for`Text-Based Style Editing`.
|
||||||
- [2024.03]: We optimize the training UI and checkpoint management. New [LAR-Gen](https://arxiv.org/abs/2403.19534) model has been added on SCEPTER Studio, supporting `zoom-out`, `virtual try on`, `inpainting`.
|
- [2024.03]: We optimize the training UI and checkpoint management. New [LAR-Gen](https://arxiv.org/abs/2403.19534) model has been added on SCEPTER Studio, supporting `zoom-out`, `virtual try on`, `inpainting`.
|
||||||
@@ -28,47 +36,162 @@ SCEPTER offers 3 core components:
|
|||||||
- [2023.12]: We release [🪄SCEPTER](https://github.com/modelscope/scepter/) library.
|
- [2023.12]: We release [🪄SCEPTER](https://github.com/modelscope/scepter/) library.
|
||||||
|
|
||||||
|
|
||||||
## 🖼 Gallery for Recent Works
|
|
||||||
|
|
||||||
### Edit Tuners
|
|
||||||
|
|
||||||
Simply provide 10 image pairs, SCEPTER will tune an edit tuner for your own Image-to-Image tasks, like `Clay Style`, `De-Text`, `Segmentation`, etc.
|
## 🪄ACE
|
||||||
Try our official few-shot datasets: [De-Text](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fdetext.zip), [Image2Hed](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fhed_pair.zip), [Image2Depth](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fimage2depth.zip), [Depth2Image](https://www.modelscope.cn/api/v1/models/iic/scepter/repo?Revision=master&FilePath=datasets%2Fdepth2image.zip).
|
|
||||||
|
|
||||||
|
ACE is a unified foundational model framework that supports a wide range of visual generation tasks. By defining CU for unifying multi-modal inputs across different tasks and incorporating long-context CU, we introduce historical contextual information into visual generation tasks, paving the way for ChatGPT-like dialog systems in visual generation.
|
||||||
|
|
||||||
|
[](https://ali-vilab.github.io/ace-page/)
|
||||||
|
|
||||||
|
### ACE Models
|
||||||
|
| **Model** | **Status** |
|
||||||
|
|:----------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
|
||||||
|
| ACE-0.6B-512px | [](https://huggingface.co/spaces/scepter-studio/ACE-Chat)<br>[](https://www.modelscope.cn/models/iic/ACE-0.6B-512px) [](https://huggingface.co/scepter-studio/ACE-0.6B-512px) |
|
||||||
|
| ACE-0.6B-1024px | [](https://huggingface.co/spaces/scepter-studio/ACE-Refiner-Chat)<br>[](https://www.modelscope.cn/models/iic/ACE-0.6B-1024px) [](https://huggingface.co/scepter-studio/ACE-0.6B-1024px) | |
|
||||||
|
| ACE-12B-FLUX-dev | Coming Soon |
|
||||||
|
### ACE Training
|
||||||
|
|
||||||
|
We offer a demonstration training YAML that enables the end-to-end training of ACE using a toy dataset. For a comprehensive overview of the hyperparameter configurations, please consult `scepter/methods/edit/dit_ace_0.6b_512.yaml`.
|
||||||
|
|
||||||
|
#### Prepare datasets
|
||||||
|
|
||||||
|
Please find the dataset class located in `scepter/modules/data/dataset/ms_dataset.py`,
|
||||||
|
designed to facilitate end-to-end training using an open-source toy dataset.
|
||||||
|
Download a dataset zip file from [modelscope](https://www.modelscope.cn/models/iic/scepter/resolve/master/datasets/hed_pair.zip), and then extract its contents into the `cache/datasets/` directory.
|
||||||
|
|
||||||
|
Should you wish to prepare your own datasets, we recommend consulting `scepter/modules/data/dataset/ms_dataset.py` for detailed guidance on the required data format.
|
||||||
|
|
||||||
|
#### Prepare initial weight
|
||||||
|
The ACE checkpoint has been uploaded to both ModelScope and HuggingFace platforms:
|
||||||
|
* [ModelScope](https://www.modelscope.cn/models/iic/ACE-0.6B-512px)
|
||||||
|
* [HuggingFace](https://huggingface.co/scepter-studio/ACE-0.6B-512px)
|
||||||
|
|
||||||
|
In the provided training YAML configuration, we have designated the Modelscope URL as the default checkpoint URL. Should you wish to transition to Hugging Face, you can effortlessly achieve this by modifying the PRETRAINED_MODEL value within the YAML file (replace the prefix "ms://iic" to "hf://scepter-studio").
|
||||||
|
|
||||||
|
|
||||||
|
#### Start training
|
||||||
|
|
||||||
|
You can easily start training procedure by executing the following command:
|
||||||
|
```bash
|
||||||
|
# ACE-0.6B-512px
|
||||||
|
PYTHONPATH=. python scepter/tools/run_train.py --cfg scepter/methods/edit/dit_ace_0.6b_512.yaml
|
||||||
|
# ACE-0.6B-1024px
|
||||||
|
PYTHONPATH=. python scepter/tools/run_train.py --cfg scepter/methods/edit/dit_ace_0.6b_1024.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
### ACE Chat Bot
|
||||||
|
|
||||||
|
We have developed a chatbot interface utilizing Gradio, designed to convert user input in natural language into visually captivating images that align semantically with the specified instructions. You can easily access this functionality by launching Scepter Studio with the following command:
|
||||||
|
```bash
|
||||||
|
PYTHONPATH=. python scepter/tools/webui.py --cfg scepter/methods/studio/scepter_ui.yaml --language zh --tab chatbot
|
||||||
|
```
|
||||||
|
Upon starting, you will find a "ChatBot" tab within the Gradio application, which serves as a chat-based interface to handle any requests related to image editing or generation.
|
||||||
|
|
||||||
|
### ACE ComfyUI Workflow
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
<table><tbody>
|
<table><tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<th align="center" colspan="4">Clay Style<br>Prompt: "Convert this image into clay style"</th>
|
<th align="center" colspan="4">ACE Workflow Examples</th>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><img src="asset/images/edit_tuner/vermeer.jpeg" width="300"></td>
|
<th align="center" colspan="1">Control</th>
|
||||||
<td><img src="asset/images/edit_tuner/clay_vermeer.jpeg" width="300"></td>
|
<th align="center" colspan="1">Semantic</th>
|
||||||
<td><img src="asset/images/edit_tuner/cat_512.jpg" width="300"></td>
|
<th align="center" colspan="1">Element</th>
|
||||||
<td><img src="asset/images/edit_tuner/clay_cat.jpeg" width="300"></td>
|
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<th align="center" colspan="2">De-Text<br>Prompt: "Remove the texts"</th>
|
<td>
|
||||||
<th align="center" colspan="2">Image2Hed<br>Prompt: "Convert to an edge map"</th>
|
<a href="https://github.com/ali-vilab/ace-page/raw/main/assets/comfyui/ace_control.png" target="_blank">
|
||||||
</tr>
|
<img src="https://github.com/ali-vilab/ace-page/raw/main/assets/comfyui/ace_control.png" width="200">
|
||||||
<tr>
|
</a>
|
||||||
<td><img src="asset/images/edit_tuner/text.jpg" width="300"></td>
|
</td>
|
||||||
<td><img src="asset/images/edit_tuner/detext.jpeg" width="300"></td>
|
<td>
|
||||||
<td><img src="asset/images/edit_tuner/cat_512.jpg" width="300"></td>
|
<a href="https://github.com/ali-vilab/ace-page/raw/main/assets/comfyui/ace_semantic.png" target="_blank">
|
||||||
<td><img src="asset/images/edit_tuner/hed.jpeg" width="300"></td>
|
<img src="https://github.com/ali-vilab/ace-page/raw/main/assets/comfyui/ace_semantic.png" width="200">
|
||||||
</tr>
|
</a>
|
||||||
<tr>
|
</td>
|
||||||
<th align="center" colspan="2">Image2Depth<br>Prompt: "Calculate the depth map"</th>
|
<td>
|
||||||
<th align="center" colspan="2">Depth2Image<br>Prompt: "Convert depth map into color image"</th>
|
<a href="https://github.com/ali-vilab/ace-page/raw/main/assets/comfyui/ace_element.png" target="_blank">
|
||||||
</tr>
|
<img src="https://github.com/ali-vilab/ace-page/raw/main/assets/comfyui/ace_element.png" width="200">
|
||||||
<tr>
|
</a>
|
||||||
<td><img src="asset/images/edit_tuner/house.jpg" width="300"></td>
|
</td>
|
||||||
<td><img src="asset/images/edit_tuner/image2depth.jpeg" width="300"></td>
|
|
||||||
<td><img src="asset/images/edit_tuner/depth.jpg" width="300"></td>
|
|
||||||
<td><img src="asset/images/edit_tuner/depth2image.jpeg" width="300"></td>
|
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
Note: Left image is input and right image is output.
|
|
||||||
|
## 🖼 Gallery for Recent Works
|
||||||
|
|
||||||
|
### FLUX Tuners
|
||||||
|
|
||||||
|
<table><tbody>
|
||||||
|
<tr>
|
||||||
|
<th align="center" colspan="3">Yarn Style</th>
|
||||||
|
<th align="center" colspan="3">Soft Watercolor Style</th>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_2_1.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_2_2.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_2_3.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_1_1.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_1_2.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_1_3.webp" width="200"></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th align="center" colspan="3">Travel Style</th>
|
||||||
|
<th align="center" colspan="3">WuKong Style</th>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_3_1.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_3_2.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_3_3.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_4_1.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_4_2.webp" width="200"></td>
|
||||||
|
<td><img src="asset/images/flux_tuner/flux_tuner_4_3.webp" width="200"></td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
### ComfyUI Workflow
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<table><tbody>
|
||||||
|
<tr>
|
||||||
|
<th align="center" colspan="4">Example Workflow Case</th>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th align="center" colspan="1">Base</th>
|
||||||
|
<th align="center" colspan="1">+Mantra</th>
|
||||||
|
<th align="center" colspan="1">+Tuner</th>
|
||||||
|
<th align="center" colspan="1">+Control</th>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<a href="asset/workflow/sdxl_base.json" target="_blank">
|
||||||
|
<img src="asset/workflow/sdxl_base.jpg" width="200">
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href="asset/workflow/sdxl_base_mantra.json" target="_blank">
|
||||||
|
<img src="asset/workflow/sdxl_base_mantra.jpg" width="200">
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href="asset/workflow/sdxl_base_mantra_tuner.json" target="_blank">
|
||||||
|
<img src="asset/workflow/sdxl_base_mantra_tuner.jpg" width="200">
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href="asset/workflow/sdxl_base_mantra_tuner_control.json" target="_blank">
|
||||||
|
<img src="asset/workflow/sdxl_base_mantra_tuner_control.jpg" width="200">
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
|
||||||
## 🛠️ Installation
|
## 🛠️ Installation
|
||||||
|
|
||||||
@@ -102,16 +225,18 @@ pip install scepter
|
|||||||
|
|
||||||
### Currently supported approaches
|
### Currently supported approaches
|
||||||
|
|
||||||
| Tasks | Methods | Links |
|
| Tasks | Methods | Links |
|
||||||
|:----------------------------:|:--------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|:----------------------------:|:----------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| Text-to-image generation | SD v1.5 | [](https://huggingface.co/runwayml/stable-diffusion-v1-5) |
|
| Text-to-image Generation | SD v1.5 | [](https://huggingface.co/runwayml/stable-diffusion-v1-5) |
|
||||||
| Text-to-image generation | SD v2.1 | [](https://huggingface.co/runwayml/stable-diffusion-v1-5) |
|
| Text-to-image Generation | SD v2.1 | [](https://huggingface.co/runwayml/stable-diffusion-v1-5) |
|
||||||
| Text-to-image generation | SD-XL | [](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) |
|
| Text-to-image Generation | SD-XL | [](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) |
|
||||||
| Efficient Tuning | LoRA | [](https://arxiv.org/abs/2106.09685) |
|
| Text-to-image Generation | FLUX | [](https://huggingface.co/black-forest-labs/FLUX.1-dev) |
|
||||||
| Efficient Tuning | Res-Tuning(NeurIPS23) | [](https://arxiv.org/abs/2310.19859) [](https://res-tuning.github.io/) |
|
| Efficient Tuning | LoRA | [](https://arxiv.org/abs/2106.09685) |
|
||||||
| Controllable image synthesis | [🌟SCEdit(CVPR24)](docs/en/tasks/scedit.md) | [](https://arxiv.org/abs/2312.11392) [](https://scedit.github.io/) |
|
| Efficient Tuning | Res-Tuning(NeurIPS23) | [](https://arxiv.org/abs/2310.19859) [](https://res-tuning.github.io/) |
|
||||||
| Image editing | [🌟LAR-Gen](docs/en/tasks/largen.md) | [](https://arxiv.org/abs/2403.19534) [](https://ali-vilab.github.io/largen-page/) |
|
| Controllable Image Synthesis | [🌟SCEdit(CVPR24)](docs/en/tasks/scedit.md) | [](https://arxiv.org/abs/2312.11392) [](https://scedit.github.io/) |
|
||||||
| Image editing | [🌟StyleBooth](docs/en/tasks/stylebooth.md) | [](https://arxiv.org/abs/2404.12154) [](https://ali-vilab.github.io/stylebooth-page/) |
|
| Image Editing | [🌟LAR-Gen](docs/en/tasks/largen.md) | [](https://arxiv.org/abs/2403.19534) [](https://ali-vilab.github.io/largen-page/) |
|
||||||
|
| Image Editing | [🌟StyleBooth](docs/en/tasks/stylebooth.md) | [](https://arxiv.org/abs/2404.12154) [](https://ali-vilab.github.io/stylebooth-page/) |
|
||||||
|
| Image Generation and Editing | [🌟ACE](https://ali-vilab.github.io/ace-page/) | [](https://arxiv.org/abs/2410.00086) [](https://ali-vilab.github.io/ace-page/) [](https://huggingface.co/spaces/scepter-studio/ACE-Chat) <br> [](https://www.modelscope.cn/models/iic/ACE-0.6B-512px) [](https://huggingface.co/scepter-studio/ACE-0.6B-512px) |
|
||||||
|
|
||||||
|
|
||||||
## 🖥️ SCEPTER Studio
|
## 🖥️ SCEPTER Studio
|
||||||
@@ -145,6 +270,24 @@ Therefore, subsequent startups will become much faster (about one minute) as dow
|
|||||||
We deploy a work studio on Modelscope that includes only the inference tab, please refer to [ms_scepter_studio](https://www.modelscope.cn/studios/iic/scepter_studio/summary) and [hf_scepter_studio](https://huggingface.co/spaces/modelscope/scepter_studio)
|
We deploy a work studio on Modelscope that includes only the inference tab, please refer to [ms_scepter_studio](https://www.modelscope.cn/studios/iic/scepter_studio/summary) and [hf_scepter_studio](https://huggingface.co/spaces/modelscope/scepter_studio)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## ⚙️️ ComfyUI Workflow
|
||||||
|
|
||||||
|
We support the use of all models in the ComfyUI Workflow through the following methods:
|
||||||
|
|
||||||
|
1) Automatic installation directly via the ComfyUI Manager by searching for the **ComfyUI-Scepter** node.
|
||||||
|
2) Manually install by moving custom_nodes from Scepter to ComfyUI.
|
||||||
|
```shell
|
||||||
|
git clone https://github.com/modelscope/scepter.git
|
||||||
|
cd path/to/scepter
|
||||||
|
pip install -e .
|
||||||
|
cp -r path/to/scepter/workflow/ path/to/ComfyUI/custom_nodes/ComfyUI-Scepter
|
||||||
|
cd path/to/ComfyUI
|
||||||
|
python main.py
|
||||||
|
```
|
||||||
|
|
||||||
|
**Note**: You can use the nodes by dragging the sample images into ComfyUI. Additionally, our nodes can automatically pull models from ModelScope or HuggingFace by selecting the *model_source* field, or you can place the already downloaded models in a local path.
|
||||||
|
|
||||||
## 🔍 Learn More
|
## 🔍 Learn More
|
||||||
|
|
||||||
- [Alibaba TongYi Vision Intelligence Lab](https://github.com/ali-vilab)
|
- [Alibaba TongYi Vision Intelligence Lab](https://github.com/ali-vilab)
|
||||||
@@ -177,4 +320,4 @@ This project is licensed under the [Apache License (Version 2.0)](https://github
|
|||||||
|
|
||||||
|
|
||||||
## Acknowledgement
|
## Acknowledgement
|
||||||
Thanks to [Stability-AI](https://github.com/Stability-AI), [SWIFT library](https://github.com/modelscope/swift/) and [Fooocus](https://github.com/lllyasviel/Fooocus) for their awesome work.
|
Thanks to [Stability-AI](https://github.com/Stability-AI), [SWIFT library](https://github.com/modelscope/swift/), [Fooocus](https://github.com/lllyasviel/Fooocus) and [ComfyUI](https://github.com/comfyanonymous/ComfyUI) for their awesome work.
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
albumentations
|
albumentations
|
||||||
|
beautifulsoup4
|
||||||
bezier
|
bezier
|
||||||
einops
|
einops
|
||||||
modelscope
|
modelscope[framework]<=1.20.1
|
||||||
ms-swift>=2.0.1
|
ms-swift
|
||||||
numpy
|
numpy
|
||||||
open_clip_torch
|
open_clip_torch
|
||||||
opencv-python
|
opencv-python
|
||||||
@@ -11,5 +12,7 @@ oss2>=2.15.0
|
|||||||
pycocotools
|
pycocotools
|
||||||
pyyaml>=5.3.1
|
pyyaml>=5.3.1
|
||||||
scikit-image
|
scikit-image
|
||||||
|
scikit-learn
|
||||||
|
sentencepiece
|
||||||
torchsde
|
torchsde
|
||||||
transformers
|
transformers<=4.46.3
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
git+https://github.com/cocodataset/panopticapi.git
|
git+https://github.com/cocodataset/panopticapi.git
|
||||||
torch==2.0.1
|
torch==2.4.1
|
||||||
torchvision==0.15.2
|
torchvision==.19.1
|
||||||
xformers==0.0.21
|
flash-attn==2.5.8
|
||||||
|
xformers==0.0.28
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
bitsandbytes
|
bitsandbytes<=0.44.1
|
||||||
gradio>=3.47.1,<4.0.0
|
gradio
|
||||||
|
gradio_imageslider
|
||||||
imagehash
|
imagehash
|
||||||
psutil
|
psutil
|
||||||
tiktoken
|
tiktoken
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 2024
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: ACESolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 500
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: 50
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: ./cache/save_data/ace_0.6b_1024
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
- NAME: "HuggingfaceFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
- NAME: "LocalFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
- NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionACE
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: True
|
||||||
|
EVAL_EMA: False
|
||||||
|
TEXT_IDENTIFIER: [ '{image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
USE_TEXT_POS_EMBEDDINGS: True
|
||||||
|
#
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: eps
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: LinearScheduler
|
||||||
|
NUM_TIMESTEPS: 1000
|
||||||
|
BETA_MIN: 0.0001
|
||||||
|
BETA_MAX: 0.02
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: ACE
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/dit/ace_0.6b_1024px.pth
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
Y_CHANNELS: 4096
|
||||||
|
MAX_SEQ_LEN: 4096
|
||||||
|
QK_NORM: True
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
ATTENTION_BACKEND: flash_attn
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
EMBED_DIM: 4
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/vae/vae.bin
|
||||||
|
IGNORE_KEYS: []
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/text_encoder/t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://iic/ACE-0.6B-1024px@models/tokenizer/t5-v1_1-xxl
|
||||||
|
LENGTH: 120
|
||||||
|
T5_DTYPE: bfloat16
|
||||||
|
ADDED_IDENTIFIER: [ '{image}', '{caption}', '{mask}', '{ref_image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 20
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 1e-7
|
||||||
|
EPS: 1e-10
|
||||||
|
WEIGHT_DECAY: 5e-4
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDatasetForACE
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: cache/datasets/hed_pair
|
||||||
|
MS_DATASET_NAMESPACE: ""
|
||||||
|
MS_DATASET_SPLIT: "train"
|
||||||
|
MS_DATASET_SUBNAME: ""
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
MAX_SEQ_LEN: 4096
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 1
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 0
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 50
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 100
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 2024
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: ACESolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 500
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: 50
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: ./cache/save_data/ace_0.6b_512
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
- NAME: "HuggingfaceFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
- NAME: "LocalFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
- NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionACE
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: True
|
||||||
|
EVAL_EMA: False
|
||||||
|
TEXT_IDENTIFIER: [ '{image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
USE_TEXT_POS_EMBEDDINGS: True
|
||||||
|
#
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: eps
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: LinearScheduler
|
||||||
|
NUM_TIMESTEPS: 1000
|
||||||
|
BETA_MIN: 0.0001
|
||||||
|
BETA_MAX: 0.02
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: ACE
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-512px@models/dit/ace_0.6b_512px.pth
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
Y_CHANNELS: 4096
|
||||||
|
MAX_SEQ_LEN: 1024
|
||||||
|
QK_NORM: True
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
ATTENTION_BACKEND: flash_attn
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
EMBED_DIM: 4
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-512px@models/vae/vae.bin
|
||||||
|
IGNORE_KEYS: []
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-512px@models/text_encoder/t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://iic/ACE-0.6B-512px@models/tokenizer/t5-v1_1-xxl
|
||||||
|
LENGTH: 120
|
||||||
|
T5_DTYPE: bfloat16
|
||||||
|
ADDED_IDENTIFIER: [ '{image}', '{caption}', '{mask}', '{ref_image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 20
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 1e-7
|
||||||
|
EPS: 1e-10
|
||||||
|
WEIGHT_DECAY: 5e-4
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDatasetForACE
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: cache/datasets/hed_pair
|
||||||
|
MS_DATASET_NAMESPACE: ""
|
||||||
|
MS_DATASET_SPLIT: "train"
|
||||||
|
MS_DATASET_SUBNAME: ""
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
MAX_SEQ_LEN: 1024
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 1
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 0
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 50
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 100
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
@@ -20,7 +20,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ SOLVER:
|
|||||||
# NUM_FOLDS DESCRIPTION: Num folds for training. TYPE: int default: 0
|
# NUM_FOLDS DESCRIPTION: Num folds for training. TYPE: int default: 0
|
||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
# WORK_DIR DESCRIPTION: Save dir of the training log or model. TYPE: str default: ''
|
# WORK_DIR DESCRIPTION: Save dir of the training log or model. TYPE: str default: ''
|
||||||
WORK_DIR: ./exp12/
|
WORK_DIR: ./cache/save_data/example/
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
# EVAL_INTERVAL DESCRIPTION: Eval the model interval. TYPE: int default: 1
|
# EVAL_INTERVAL DESCRIPTION: Eval the model interval. TYPE: int default: 1
|
||||||
EVAL_INTERVAL: 1
|
EVAL_INTERVAL: 1
|
||||||
@@ -102,7 +102,7 @@ SOLVER:
|
|||||||
# DATASET DESCRIPTION: the public dataset name TYPE: str default: 'cifar10'
|
# DATASET DESCRIPTION: the public dataset name TYPE: str default: 'cifar10'
|
||||||
DATASET: cifar10
|
DATASET: cifar10
|
||||||
# DATA_ROOT DESCRIPTION: the download data save path TYPE: str default: ''
|
# DATA_ROOT DESCRIPTION: the download data save path TYPE: str default: ''
|
||||||
DATA_ROOT: ./local_data/cifar10
|
DATA_ROOT: ./cache/cache_data/cifar10
|
||||||
# MODE DESCRIPTION: test TYPE: str default: test
|
# MODE DESCRIPTION: test TYPE: str default: test
|
||||||
MODE: test
|
MODE: test
|
||||||
# PIN_MEMORY DESCRIPTION: pin_memory for data loader TYPE: bool default: False
|
# PIN_MEMORY DESCRIPTION: pin_memory for data loader TYPE: bool default: False
|
||||||
|
|||||||
@@ -0,0 +1,235 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 42
|
||||||
|
TENSOR_PARALLEL_SIZE: 1
|
||||||
|
PIPELINE_PARALLEL_SIZE: 1
|
||||||
|
SYS_ENVS:
|
||||||
|
TORCH_CUDNN_V8_API_ENABLED: '1'
|
||||||
|
TOKENIZERS_PARALLELISM: 'false'
|
||||||
|
TF_CPP_MIN_LOG_LEVEL: '3'
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionVideoSolver
|
||||||
|
MAX_STEPS: 2000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_cogvideox_2b_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 4
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
FPS: 8
|
||||||
|
SHARDING_STRATEGY: full_shard
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
SAVE_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 64
|
||||||
|
LORA_ALPHA: 64
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.to_k|.to_q|.to_v|.to_out.0)$"
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionCogVideoX
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
PARAMETERIZATION: v
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA: 3.0
|
||||||
|
ZERO_TERMINAL_SNR: True
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
SCALING_FACTOR_IMAGE: 1.15258426
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 3.0
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@transformer/diffusion_pytorch_model.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 30
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 30
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: False
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: False
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@vae/diffusion_pytorch_model.safetensors
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SEED: 42
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
NUM_FRAMES: 49
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: Adam
|
||||||
|
LEARNING_RATE: 1e-3
|
||||||
|
BETAS: [ 0.9, 0.95 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 0.0
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
# LR_SCHEDULER:
|
||||||
|
# NAME: StepAnnealingLR
|
||||||
|
# WARMUP_STEPS: 200
|
||||||
|
# TOTAL_STEPS: 2000
|
||||||
|
# DECAY_MODE: 'cosine'
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: VideoGenDataset
|
||||||
|
MODE: train
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
PROMPT_PREFIX: 'DISNEY '
|
||||||
|
SAMPLER:
|
||||||
|
NAME: MixtureOfSamplers
|
||||||
|
SUB_SAMPLERS:
|
||||||
|
- NAME: MultiLevelBatchSampler
|
||||||
|
PROB: 1.0
|
||||||
|
FIELDS: [ "video_path", "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PATH_PREFIX: cache/datasets/Disney-VideoGeneration-Dataset/
|
||||||
|
INDEX_FILE: cache/datasets/Disney-VideoGeneration-Dataset/index.jsonl
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'video', "prompt" ]
|
||||||
|
META_KEYS: [ ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "A girl riding a bike.", "A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. The panda's fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, casting a gentle glow on the scene. The panda's face is expressive, showing concentration and joy as it plays. The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical atmosphere of this unique musical performance." ]
|
||||||
|
IMAGE_SIZE: [ 480, 720 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: 'DISNEY '
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
USE_NUM: 8
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
PRIORITY: 20
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 1000
|
||||||
|
PRIORITY: 40
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 42
|
||||||
|
TENSOR_PARALLEL_SIZE: 1
|
||||||
|
PIPELINE_PARALLEL_SIZE: 1
|
||||||
|
SYS_ENVS:
|
||||||
|
TORCH_CUDNN_V8_API_ENABLED: '1'
|
||||||
|
TOKENIZERS_PARALLELISM: 'false'
|
||||||
|
TF_CPP_MIN_LOG_LEVEL: '3'
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionVideoSolver
|
||||||
|
MAX_STEPS: 2000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_cogvideox_5b_i2v_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 4
|
||||||
|
FPS: 8
|
||||||
|
SHARDING_STRATEGY: full_shard
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
SAVE_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 64
|
||||||
|
LORA_ALPHA: 64
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.to_k|.to_q|.to_v|.to_out.0)$"
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionCogVideoX
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
PARAMETERIZATION: v
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA: 3.0
|
||||||
|
ZERO_TERMINAL_SNR: True
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
SCALING_FACTOR_IMAGE: 0.7 # 5b diff
|
||||||
|
NOISED_IMAGE_DROPOUT: 0.05
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 1.0 # 5b diff
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: # 5b-I2V diff
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b-I2V@transformer/diffusion_pytorch_model-00001-of-00003.safetensors
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b-I2V@transformer/diffusion_pytorch_model-00002-of-00003.safetensors
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b-I2V@transformer/diffusion_pytorch_model-00003-of-00003.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 48 # 5b diff
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 32 # 5b-I2V diff
|
||||||
|
LATENT_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 42 # 5b diff
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: True # 5b diff
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: True # 5b-I2V diff
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-5b-I2V@vae/diffusion_pytorch_model.safetensors # 5b diff
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SEED: 42
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
NUM_FRAMES: 49
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: Adam
|
||||||
|
LEARNING_RATE: 1e-3
|
||||||
|
BETAS: [ 0.9, 0.95 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 0.0
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
# LR_SCHEDULER:
|
||||||
|
# NAME: StepAnnealingLR
|
||||||
|
# WARMUP_STEPS: 200
|
||||||
|
# TOTAL_STEPS: 2000
|
||||||
|
# DECAY_MODE: 'cosine'
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: VideoGenDataset
|
||||||
|
MODE: train
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 0
|
||||||
|
PROMPT_PREFIX: 'DISNEY '
|
||||||
|
DATA_TYPE: 'i2v'
|
||||||
|
SAMPLER:
|
||||||
|
NAME: MixtureOfSamplers
|
||||||
|
SUB_SAMPLERS:
|
||||||
|
- NAME: MultiLevelBatchSampler
|
||||||
|
PROB: 1.0
|
||||||
|
FIELDS: [ "video_path", "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PATH_PREFIX: cache/datasets/Disney-VideoGeneration-Dataset/
|
||||||
|
INDEX_FILE: cache/datasets/Disney-VideoGeneration-Dataset/index.jsonl
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ "video", "image", "prompt" ]
|
||||||
|
META_KEYS: [ ]
|
||||||
|
#
|
||||||
|
# EVAL_DATA:
|
||||||
|
# NAME: Text2ImageDataset
|
||||||
|
# MODE: eval
|
||||||
|
# PROMPT_FILE:
|
||||||
|
# PROMPT_DATA: [ "A cat running.#;#asset/images/edit_tuner/cat_512.jpg" ]
|
||||||
|
# FIELDS: [ "prompt", "img_path" ]
|
||||||
|
# DELIMITER: '#;#'
|
||||||
|
# PROMPT_PREFIX: ''
|
||||||
|
# PIN_MEMORY: True
|
||||||
|
# BATCH_SIZE: 1
|
||||||
|
# USE_NUM: 8
|
||||||
|
# NUM_WORKERS: 0
|
||||||
|
# IMAGE_SIZE: [ 480, 720 ]
|
||||||
|
# TRANSFORMS:
|
||||||
|
# - NAME: LoadImageFromFileList
|
||||||
|
# FILE_KEYS: [ 'img_path' ]
|
||||||
|
# RGB_ORDER: RGB
|
||||||
|
# BACKEND: pillow
|
||||||
|
# - NAME: FlexibleResize
|
||||||
|
# INTERPOLATION: bilinear
|
||||||
|
# SIZE: [ 480, 720 ]
|
||||||
|
# INPUT_KEY: [ 'img' ]
|
||||||
|
# OUTPUT_KEY: [ 'img' ]
|
||||||
|
# BACKEND: pillow
|
||||||
|
# - NAME: FlexibleCenterCrop
|
||||||
|
# SIZE: [ 480, 720 ]
|
||||||
|
# INPUT_KEY: [ 'img' ]
|
||||||
|
# OUTPUT_KEY: [ 'img' ]
|
||||||
|
# BACKEND: pillow
|
||||||
|
# - NAME: ImageToTensor
|
||||||
|
# INPUT_KEY: [ 'img' ]
|
||||||
|
# OUTPUT_KEY: [ 'img' ]
|
||||||
|
# BACKEND: pillow
|
||||||
|
# - NAME: Normalize
|
||||||
|
# MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
# STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
# INPUT_KEY: [ 'img' ]
|
||||||
|
# OUTPUT_KEY: [ 'image' ]
|
||||||
|
# BACKEND: torchvision
|
||||||
|
# - NAME: Select
|
||||||
|
# KEYS: [ 'image', 'prompt' ]
|
||||||
|
# META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
PRIORITY: 20
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 1000
|
||||||
|
PRIORITY: 40
|
||||||
|
#
|
||||||
|
# EVAL_HOOKS:
|
||||||
|
# - NAME: ProbeDataHook
|
||||||
|
# PROB_INTERVAL: 100
|
||||||
|
# PRIORITY: 0
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 42
|
||||||
|
TENSOR_PARALLEL_SIZE: 1
|
||||||
|
PIPELINE_PARALLEL_SIZE: 1
|
||||||
|
SYS_ENVS:
|
||||||
|
TORCH_CUDNN_V8_API_ENABLED: '1'
|
||||||
|
TOKENIZERS_PARALLELISM: 'false'
|
||||||
|
TF_CPP_MIN_LOG_LEVEL: '3'
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionVideoSolver
|
||||||
|
MAX_STEPS: 2000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_cogvideox_5b_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 4
|
||||||
|
FPS: 8
|
||||||
|
SHARDING_STRATEGY: full_shard
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
SAVE_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 64
|
||||||
|
LORA_ALPHA: 64
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.to_k|.to_q|.to_v|.to_out.0)$"
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionCogVideoX
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
PARAMETERIZATION: v
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA: 3.0
|
||||||
|
ZERO_TERMINAL_SNR: True
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
SCALING_FACTOR_IMAGE: 0.7 # 5b diff
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 1.0 # 5b diff
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: # 5b diff
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b@transformer/diffusion_pytorch_model-00001-of-00002.safetensors
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b@transformer/diffusion_pytorch_model-00002-of-00002.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 48 # 5b diff
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 42 # 5b diff
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: True # 5b diff
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: False
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-5b@vae/diffusion_pytorch_model.safetensors # 5b diff
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SEED: 42
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
NUM_FRAMES: 49
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: Adam
|
||||||
|
LEARNING_RATE: 1e-3
|
||||||
|
BETAS: [ 0.9, 0.95 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 0.0
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
# LR_SCHEDULER:
|
||||||
|
# NAME: StepAnnealingLR
|
||||||
|
# WARMUP_STEPS: 200
|
||||||
|
# TOTAL_STEPS: 2000
|
||||||
|
# DECAY_MODE: 'cosine'
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: VideoGenDatasetOTF
|
||||||
|
MODE: train
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
PROMPT_PREFIX: 'DISNEY '
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
FIELDS: [ 'video_path', 'prompt' ]
|
||||||
|
PATH_PREFIX: cache/datasets/Disney-VideoGeneration-Dataset/
|
||||||
|
DATA_FILE: cache/datasets/Disney-VideoGeneration-Dataset/index.jsonl
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'video', 'video_latent', "prompt" ]
|
||||||
|
META_KEYS: [ ]
|
||||||
|
MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-5b@vae/diffusion_pytorch_model.safetensors
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "A girl riding a bike.", "A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. The panda's fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, casting a gentle glow on the scene. The panda's face is expressive, showing concentration and joy as it plays. The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical atmosphere of this unique musical performance." ]
|
||||||
|
IMAGE_SIZE: [ 480, 720 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: 'DISNEY '
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
USE_NUM: 8
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
PRIORITY: 20
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 1000
|
||||||
|
PRIORITY: 40
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
@@ -0,0 +1,246 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 166666
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
MAX_STEPS: 100000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_flux_dev_1024_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 16
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.t5_model' ] #
|
||||||
|
SAVE_MODULES: [ 'model'] #
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
FREEZE:
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 4
|
||||||
|
LORA_ALPHA: 4
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "(model.double_blocks.*(.qkv|.proj|.img_mod.lin|.txt_mod.lin))|(model.single_blocks.*(.linear1|.linear2|.modulation.lin))$"
|
||||||
|
##
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionFlux
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
SAMPLER_SCHEDULER:
|
||||||
|
NAME: FlowMatchFluxShiftScheduler
|
||||||
|
SHIFT: False
|
||||||
|
SIGMOID_SCALE: 1
|
||||||
|
BASE_SHIFT: 0.5
|
||||||
|
MAX_SHIFT: 1.15
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: Flux
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@flux1-dev.safetensors
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
NUM_HEADS: 24
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
THETA: 10000
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
GUIDANCE_EMBED: True
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
QKV_BIAS: True
|
||||||
|
DEPTH: 19
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
T5_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder_2/
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer_2/
|
||||||
|
MAX_LENGTH: 512
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
CLIP_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder/
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer/
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SAMPLER: flow_euler
|
||||||
|
SEED: 2024
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a boy wearing a jacket", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 2
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
# GRADIENT_CLIP: 1.0
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
@@ -0,0 +1,267 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 166666
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
MAX_STEPS: 100000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_flux_schnell_1024_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 16
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.t5_model' ] #
|
||||||
|
SAVE_MODULES: [ 'model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
FREEZE:
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 4
|
||||||
|
LORA_ALPHA: 4
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "(model.double_blocks.*(.qkv|.proj|.img_mod.lin|.txt_mod.lin))|(model.single_blocks.*(.linear1|.linear2|.modulation.lin))$"
|
||||||
|
##
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionFlux
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
# WEIGHTING_SCHEME DESCRIPTION: The weighting scheme for sampling timesteps, choose from ['sigma_sqrt', 'logit_normal', 'mode', 'cosmap', 'none']. TYPE: str default: 'logit_normal'
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
# LOGIT_MEAN DESCRIPTION: The mean of the logit distribution for sampling timesteps. TYPE: float default: 0.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
# LOGIT_STD DESCRIPTION: The standard deviation of the logit distribution for sampling timesteps. TYPE: float default: 1.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
# MODE_SCALE DESCRIPTION: The scale factor for the mode of the logit distribution for sampling timesteps. TYPE: float default: 1.29
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
SAMPLER_SCHEDULER:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'FlowMatchFluxShiftScheduler'
|
||||||
|
NAME: FlowMatchFluxShiftScheduler
|
||||||
|
# SHIFT DESCRIPTION: Use timestamp shift or not, default is True. TYPE: bool default: True
|
||||||
|
SHIFT: False
|
||||||
|
# SIGMOID_SCALE DESCRIPTION: The scale of sigmoid function for sampling timesteps. TYPE: int default: 1
|
||||||
|
SIGMOID_SCALE: 1
|
||||||
|
# BASE_SHIFT DESCRIPTION: The base shift factor for the timestamp. TYPE: float default: 0.5
|
||||||
|
BASE_SHIFT: 0.5
|
||||||
|
# MAX_SHIFT DESCRIPTION: The max shift factor for the timestamp. TYPE: float default: 1.15
|
||||||
|
MAX_SHIFT: 1.15
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'Flux'
|
||||||
|
NAME: Flux
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-schnell@flux1-schnell.safetensors
|
||||||
|
# IN_CHANNELS DESCRIPTION: model's input channels. TYPE: int default: 64
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
# HIDDEN_SIZE DESCRIPTION: model's hidden size. TYPE: int default: 1024
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
# NUM_HEADS DESCRIPTION: number of heads in the transformer. TYPE: int default: 16
|
||||||
|
NUM_HEADS: 24
|
||||||
|
# AXES_DIM DESCRIPTION: dimensions of the axes of the positional encoding. TYPE: list default: [16, 56, 56]
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
# THETA DESCRIPTION: theta for positional encoding. TYPE: int default: 10000
|
||||||
|
THETA: 10000
|
||||||
|
# VEC_IN_DIM DESCRIPTION: dimension of the vector input. TYPE: int default: 768
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
# GUIDANCE_EMBED DESCRIPTION: whether to use guidance embedding. TYPE: bool default: False
|
||||||
|
GUIDANCE_EMBED: False
|
||||||
|
# CONTEXT_IN_DIM DESCRIPTION: dimension of the context input. TYPE: int default: 4096
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
# MLP_RATIO DESCRIPTION: ratio of mlp hidden size to hidden size. TYPE: float default: 4.0
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
# QKV_BIAS DESCRIPTION: whether to use bias in qkv projection. TYPE: bool default: True
|
||||||
|
QKV_BIAS: True
|
||||||
|
# DEPTH DESCRIPTION: number of transformer blocks. TYPE: int default: 19
|
||||||
|
DEPTH: 19
|
||||||
|
# DEPTH_SINGLE_BLOCKS DESCRIPTION: number of transformer blocks in the single stream block. TYPE: int default: 38
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-schnell@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
T5_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-schnell@text_encoder_2/
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-schnell@tokenizer_2/
|
||||||
|
MAX_LENGTH: 256
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
CLIP_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-schnell@text_encoder/
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-schnell@tokenizer/
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLE_STEPS: 4
|
||||||
|
SAMPLER: flow_euler
|
||||||
|
SEED: 2024
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a boy wearing a jacket", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 2
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
# GRADIENT_CLIP: 1.0
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 1000
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: ./cache/save_data/dit_pixart_alpha_1024_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
FREEZE:
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 128
|
||||||
|
LORA_ALPHA: 128
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.q|.k|.v|.o|mlp.fc1|mlp.fc2)$"
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionPixart
|
||||||
|
PARAMETERIZATION: eps
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "linear"
|
||||||
|
"BETA_MIN": 0.0001
|
||||||
|
"BETA_MAX": 0.02
|
||||||
|
USE_EMA: False
|
||||||
|
LOAD_REFINER: False
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: PixArt
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@PixArt-XL-2-1024-MS.pth
|
||||||
|
INPUT_SIZE: 128
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
CLASS_DROPOUT_PROB: 0.1
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
USE_REL_POS: False
|
||||||
|
CAPTION_CHANNELS: 4096
|
||||||
|
LEWEI_SCALE: 2
|
||||||
|
MODEL_MAX_LENGTH: 120
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-2-base@512-base-ema.safetensors
|
||||||
|
EMBED_DIM: 4
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/
|
||||||
|
LENGTH: 120
|
||||||
|
CLEAN: heavy
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 20
|
||||||
|
SEED: 2024
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
RUN_TRAIN_N: False
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 0.0001
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a boy wearing a jacket", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 0
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
SHOW_GPU_MEM: True
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 500
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: 50
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: ./cache/save_data/dit_sd3_1024
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
- NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionSD3
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "shifted"
|
||||||
|
"SHIFT": 3
|
||||||
|
USE_EMA: False
|
||||||
|
T_WEIGHT: uniform
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: MMDiT
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
IGNORE_KEYS: '^first_stage_model.'
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
DEPTH: 24
|
||||||
|
INPUT_SIZE:
|
||||||
|
ADM_IN_CHANNELS: 2048
|
||||||
|
CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } }
|
||||||
|
NUM_PATCHES: 36864
|
||||||
|
POS_EMBED_MAX_SIZE: 192
|
||||||
|
POS_EMBED_SCALING_FACTOR:
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
EMBED_DIM: 16
|
||||||
|
IGNORE_KEYS: '^model.diffusion_model.'
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
USE_CONV: False
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: SD3TextEmbedder
|
||||||
|
P_ZERO: 0.0
|
||||||
|
CLIP_L:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
CLIP_G:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
T5_XXL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3
|
||||||
|
LENGTH: 256
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
T5_DTYPE: float16
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: euler
|
||||||
|
SAMPLE_STEPS: 28
|
||||||
|
SEED: 1749023094
|
||||||
|
GUIDE_SCALE: 5.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
RUN_TRAIN_N: False
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 1e-5
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 10000
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
SHOW_GPU_MEM: True
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 50
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 500
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: 50
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: ./cache/save_data/dit_sd3_1024_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
- NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 128
|
||||||
|
LORA_ALPHA: 128
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.attn.qkv|.attn.proj|mlp.fc1|mlp.fc2)$"
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionSD3
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "shifted"
|
||||||
|
"SHIFT": 3
|
||||||
|
USE_EMA: False
|
||||||
|
T_WEIGHT: uniform
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: MMDiT
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
IGNORE_KEYS: '^first_stage_model.'
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
DEPTH: 24
|
||||||
|
INPUT_SIZE:
|
||||||
|
ADM_IN_CHANNELS: 2048
|
||||||
|
CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } }
|
||||||
|
NUM_PATCHES: 36864
|
||||||
|
POS_EMBED_MAX_SIZE: 192
|
||||||
|
POS_EMBED_SCALING_FACTOR:
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
EMBED_DIM: 16
|
||||||
|
IGNORE_KEYS: '^model.diffusion_model.'
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
USE_CONV: False
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: SD3TextEmbedder
|
||||||
|
P_ZERO: 0.0
|
||||||
|
CLIP_L:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
CLIP_G:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
T5_XXL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3
|
||||||
|
LENGTH: 256
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
T5_DTYPE: float16
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: euler
|
||||||
|
SAMPLE_STEPS: 28
|
||||||
|
SEED: 1749023094
|
||||||
|
GUIDE_SCALE: 5.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
RUN_TRAIN_N: False
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 5e-5
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 10000
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
SHOW_GPU_MEM: True
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 50
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_full
|
WORK_DIR: ./cache/save_data/sd15_512_full
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
MODEL:
|
MODEL:
|
||||||
NAME: LatentDiffusion
|
NAME: LatentDiffusion
|
||||||
@@ -124,7 +125,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.0064
|
LEARNING_RATE: 0.00001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -190,7 +191,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_lora
|
WORK_DIR: ./cache/save_data/sd15_512_lora
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
NAME: SwiftLoRA
|
NAME: SwiftLoRA
|
||||||
@@ -132,7 +133,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -198,7 +199,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_textlora
|
WORK_DIR: ./cache/save_data/sd15_512_textlora
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
NAME: SwiftLoRA
|
NAME: SwiftLoRA
|
||||||
@@ -140,7 +141,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -206,7 +207,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_512_full
|
WORK_DIR: ./cache/save_data/sd21_512_full
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
MODEL:
|
MODEL:
|
||||||
NAME: LatentDiffusion
|
NAME: LatentDiffusion
|
||||||
@@ -120,7 +121,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.0064
|
LEARNING_RATE: 0.00001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -186,7 +187,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_512_lora
|
WORK_DIR: ./cache/save_data/sd21_512_lora
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -129,7 +130,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.0064
|
LEARNING_RATE: 0.00001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -195,7 +196,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_768_full
|
WORK_DIR: ./cache/save_data/sd21_768_full
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
MODEL:
|
MODEL:
|
||||||
NAME: LatentDiffusion
|
NAME: LatentDiffusion
|
||||||
@@ -120,7 +121,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.0064
|
LEARNING_RATE: 0.00001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -186,7 +187,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_768_lora
|
WORK_DIR: ./cache/save_data/sd21_768_lora
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -129,7 +130,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -195,7 +196,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_full
|
WORK_DIR: ./cache/save_data/sdxl_1024_full
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
MODEL:
|
MODEL:
|
||||||
NAME: LatentDiffusionXL
|
NAME: LatentDiffusionXL
|
||||||
@@ -238,7 +239,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.0064
|
LEARNING_RATE: 0.00001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -306,7 +307,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
- NAME: Select
|
- NAME: Select
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_lora
|
WORK_DIR: ./cache/save_data/sdxl_1024_lora
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -247,7 +248,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -315,7 +316,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
- NAME: Select
|
- NAME: Select
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_textlora
|
WORK_DIR: ./cache/save_data/sdxl_1024_textlora
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -255,7 +256,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -323,7 +324,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
- NAME: Select
|
- NAME: Select
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_sce_ctr_hed
|
WORK_DIR: ./cache/save_data/sd15_512_sce_ctr_hed
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -141,7 +142,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -167,13 +168,13 @@ SOLVER:
|
|||||||
RGB_ORDER: RGB
|
RGB_ORDER: RGB
|
||||||
BACKEND: pillow
|
BACKEND: pillow
|
||||||
- NAME: Resize
|
- NAME: Resize
|
||||||
SIZE: 768
|
SIZE: 512
|
||||||
INTERPOLATION: bilinear
|
INTERPOLATION: bilinear
|
||||||
INPUT_KEY: [ 'img' ]
|
INPUT_KEY: [ 'img' ]
|
||||||
OUTPUT_KEY: [ 'img' ]
|
OUTPUT_KEY: [ 'img' ]
|
||||||
BACKEND: pillow
|
BACKEND: pillow
|
||||||
- NAME: CenterCrop
|
- NAME: CenterCrop
|
||||||
SIZE: 768
|
SIZE: 512
|
||||||
INPUT_KEY: [ 'img' ]
|
INPUT_KEY: [ 'img' ]
|
||||||
OUTPUT_KEY: [ 'img' ]
|
OUTPUT_KEY: [ 'img' ]
|
||||||
BACKEND: pillow
|
BACKEND: pillow
|
||||||
@@ -215,13 +216,13 @@ SOLVER:
|
|||||||
RGB_ORDER: RGB
|
RGB_ORDER: RGB
|
||||||
BACKEND: pillow
|
BACKEND: pillow
|
||||||
- NAME: Resize
|
- NAME: Resize
|
||||||
SIZE: 768
|
SIZE: 512
|
||||||
INTERPOLATION: bilinear
|
INTERPOLATION: bilinear
|
||||||
INPUT_KEY: [ 'img' ]
|
INPUT_KEY: [ 'img' ]
|
||||||
OUTPUT_KEY: [ 'img' ]
|
OUTPUT_KEY: [ 'img' ]
|
||||||
BACKEND: pillow
|
BACKEND: pillow
|
||||||
- NAME: CenterCrop
|
- NAME: CenterCrop
|
||||||
SIZE: 768
|
SIZE: 512
|
||||||
INPUT_KEY: [ 'img' ]
|
INPUT_KEY: [ 'img' ]
|
||||||
OUTPUT_KEY: [ 'img' ]
|
OUTPUT_KEY: [ 'img' ]
|
||||||
BACKEND: pillow
|
BACKEND: pillow
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_768_sce_ctr_canny
|
WORK_DIR: ./cache/save_data/sd21_768_sce_ctr_canny
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -140,7 +141,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_768_sce_ctr_pose
|
WORK_DIR: ./cache/save_data/sd21_768_sce_ctr_pose
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -140,7 +141,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_canny
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_canny
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -254,7 +255,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_color
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_color
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -255,7 +256,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_color_datatxt
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_color_datatxt
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -255,7 +256,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_depth
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_ctr_depth
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -255,7 +256,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_sce_t2i
|
WORK_DIR: ./cache/save_data/sd15_512_sce_t2i
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -134,7 +135,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -200,7 +201,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_sce_t2i_swift
|
WORK_DIR: ./cache/save_data/sd15_512_sce_t2i_swift
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -132,7 +133,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -198,7 +199,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd15_512_textsce_t2i_swift
|
WORK_DIR: ./cache/save_data/sd15_512_textsce_t2i_swift
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -140,7 +141,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -206,7 +207,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_768_sce_t2i
|
WORK_DIR: ./cache/save_data/sd21_768_sce_t2i
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -130,7 +131,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -196,7 +197,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sd21_768_sce_t2i_swift
|
WORK_DIR: ./cache/save_data/sd21_768_sce_t2i_swift
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -128,7 +129,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -194,7 +195,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -247,7 +248,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -315,7 +316,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
- NAME: Select
|
- NAME: Select
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i_datatxt
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i_datatxt
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
FREEZE_PART: [ "first_stage_model", "cond_stage_model", "model" ]
|
||||||
@@ -247,7 +248,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i_swift
|
WORK_DIR: ./cache/save_data/sdxl_1024_sce_t2i_swift
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -245,7 +246,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -313,7 +314,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
- NAME: Select
|
- NAME: Select
|
||||||
|
|||||||
@@ -14,13 +14,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: 100
|
EVAL_INTERVAL: 100
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR: ./cache/save_data/sdxl_1024_textsce_t2i_swift
|
WORK_DIR: ./cache/save_data/sdxl_1024_textsce_t2i_swift
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TUNER:
|
TUNER:
|
||||||
-
|
-
|
||||||
@@ -253,7 +254,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -321,7 +322,7 @@ SOLVER:
|
|||||||
NUM_WORKERS: 4
|
NUM_WORKERS: 4
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
TRANSFORMS:
|
TRANSFORMS:
|
||||||
- NAME: Select
|
- NAME: Select
|
||||||
|
|||||||
@@ -0,0 +1,25 @@
|
|||||||
|
WORK_DIR: chatbot
|
||||||
|
FILE_SYSTEM:
|
||||||
|
- NAME: LocalFs
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
- NAME: ModelscopeFs
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
- NAME: HuggingfaceFs
|
||||||
|
TEMP_DIR: ./cache/cache_data
|
||||||
|
#
|
||||||
|
ENABLE_I2V: False
|
||||||
|
SKIP_EXAMPLES: True
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
EDIT_MODEL:
|
||||||
|
MODEL_CFG_DIR: scepter/methods/studio/chatbot/models/
|
||||||
|
I2V:
|
||||||
|
MODEL_NAME: CogVideoX-5b-I2V
|
||||||
|
MODEL_DIR: ms://ZhipuAI/CogVideoX-5b-I2V/
|
||||||
|
CAPTIONER:
|
||||||
|
MODEL_NAME: InternVL2-2B
|
||||||
|
MODEL_DIR: ms://OpenGVLab/InternVL2-2B/
|
||||||
|
PROMPT: '<image>\nThis image is the first frame of a video. Based on this image, please imagine what changes may occur in the next few seconds of the video. Please output brief description, such as "a dog running" or "a person turns to left". No more than 30 words.'
|
||||||
|
ENHANCER:
|
||||||
|
MODEL_NAME: Meta-Llama-3.1-8B-Instruct
|
||||||
|
MODEL_DIR: ms://LLM-Research/Meta-Llama-3.1-8B-Instruct/
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
NAME: ACE_0.6B_1024
|
||||||
|
IS_DEFAULT: False
|
||||||
|
USE_DYNAMIC_MODEL: True
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
#
|
||||||
|
INPUT:
|
||||||
|
INPUT_IMAGE:
|
||||||
|
INPUT_MASK:
|
||||||
|
TASK:
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
OUTPUT_HEIGHT: 1024
|
||||||
|
OUTPUT_WIDTH: 1024
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
SEED: -1
|
||||||
|
TAR_INDEX: 0
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
- NAME: decode
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: forward
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE"]
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode_list_of_list
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionACE
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT: ""
|
||||||
|
TEXT_IDENTIFIER: [ '{image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
USE_TEXT_POS_EMBEDDINGS: True
|
||||||
|
#
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: eps
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: LinearScheduler
|
||||||
|
NUM_TIMESTEPS: 1000
|
||||||
|
BETA_MIN: 0.0001
|
||||||
|
BETA_MAX: 0.02
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: ACE
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/dit/ace_0.6b_1024px.pth
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
Y_CHANNELS: 4096
|
||||||
|
MAX_SEQ_LEN: 4096
|
||||||
|
QK_NORM: True
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
ATTENTION_BACKEND: flash_attn
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
EMBED_DIM: 4
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/vae/vae.bin
|
||||||
|
IGNORE_KEYS: []
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/text_encoder/t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://iic/ACE-0.6B-1024px@models/tokenizer/t5-v1_1-xxl
|
||||||
|
LENGTH: 120
|
||||||
|
T5_DTYPE: bfloat16
|
||||||
|
ADDED_IDENTIFIER: [ '{image}', '{caption}', '{mask}', '{ref_image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
NAME: ACE_0.6B_1024_REFINER
|
||||||
|
IS_DEFAULT: False
|
||||||
|
USE_DYNAMIC_MODEL: True
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
#
|
||||||
|
INPUT:
|
||||||
|
INPUT_IMAGE:
|
||||||
|
INPUT_MASK:
|
||||||
|
TASK:
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
OUTPUT_HEIGHT: 1024
|
||||||
|
OUTPUT_WIDTH: 1024
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
SEED: -1
|
||||||
|
TAR_INDEX: 0
|
||||||
|
REFINER_SCALE: 0.2
|
||||||
|
USE_ACE: True
|
||||||
|
#REFINER_PROMPT: "High Resolution, Sharpness, Clarity, Detail Enhancement, Noise Reduction, HD, 4k, Image Restoration, HDR"
|
||||||
|
REFINER_PROMPT: "High Resolution, Sharpness, Clarity, Detail Enhancement, Noise Reduction, HD, 4k, Image Restoration, HDR"
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
- NAME: decode
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: forward
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE"]
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode_list_of_list
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionACE
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT: ""
|
||||||
|
TEXT_IDENTIFIER: [ '{image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
USE_TEXT_POS_EMBEDDINGS: True
|
||||||
|
#
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: eps
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: LinearScheduler
|
||||||
|
NUM_TIMESTEPS: 1000
|
||||||
|
BETA_MIN: 0.0001
|
||||||
|
BETA_MAX: 0.02
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: ACE
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/dit/ace_0.6b_1024px.pth
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
Y_CHANNELS: 4096
|
||||||
|
MAX_SEQ_LEN: 4096
|
||||||
|
QK_NORM: True
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
ATTENTION_BACKEND: flash_attn
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
EMBED_DIM: 4
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/vae/vae.bin
|
||||||
|
IGNORE_KEYS: []
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-1024px@models/text_encoder/t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://iic/ACE-0.6B-1024px@models/tokenizer/t5-v1_1-xxl
|
||||||
|
LENGTH: 120
|
||||||
|
T5_DTYPE: bfloat16
|
||||||
|
ADDED_IDENTIFIER: [ '{image}', '{caption}', '{mask}', '{ref_image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
|
||||||
|
ACE_PROMPT: [
|
||||||
|
"A cute cartoon rabbit holding a whiteboard that says 'ACE Refiner', standing in a sunny meadow filled with flowers, with a big smile and bright colors.",
|
||||||
|
"A beautiful young woman with long flowing hair, wearing a summer dress, holding a whiteboard that reads 'ACE Refiner' while sitting on a park bench surrounded by cherry blossoms.",
|
||||||
|
"An adorable cartoon cat wearing oversized glasses, holding a whiteboard that says 'ACE Refiner', perched on a stack of colorful books in a cozy library setting.",
|
||||||
|
"A charming girl with pigtails, wearing a cute school uniform, enthusiastically holding a whiteboard that has 'ACE Refiner' written on it, in a bright and cheerful classroom full of educational posters.",
|
||||||
|
"A friendly cartoon dog with floppy ears, sitting in front of a doghouse, proudly holding a whiteboard that says 'ACE Refiner', with a playful expression and a blue sky in the background.",
|
||||||
|
"A cute anime girl with big expressive eyes, dressed in a colorful outfit, holding a whiteboard that reads 'ACE Refiner' in a fantastical landscape filled with mythical creatures.",
|
||||||
|
"A vibrant cartoon fox holding a whiteboard that says 'ACE Refiner', standing on a rock by a sparkling stream, surrounded by lush greenery and butterflies.",
|
||||||
|
"A stylish young woman in a business outfit, smiling as she holds a whiteboard written with 'ACE Refiner', in a modern office filled with plants and natural light.",
|
||||||
|
"A cute cartoon unicorn holding a sparkling whiteboard that says 'ACE Refiner', frolicking in a magical forest, with rainbows and stars in the background.",
|
||||||
|
"A happy family, consisting of a cute little girl and her playful puppy, holding a whiteboard that says 'ACE Refiner', together in their backyard on a sunny day."
|
||||||
|
]
|
||||||
|
REFINER_MODEL:
|
||||||
|
NAME: ""
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [ [ 1024, 1024 ] ]
|
||||||
|
INPUT:
|
||||||
|
INPUT_IMAGE:
|
||||||
|
INPUT_MASK:
|
||||||
|
TASK:
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
OUTPUT_HEIGHT: 1024
|
||||||
|
OUTPUT_WIDTH: 1024
|
||||||
|
SAMPLER: flow_euler
|
||||||
|
SAMPLE_STEPS: 30
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
GUIDE_RESCALE:
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: [ "IMAGE" ]
|
||||||
|
- NAME: decode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: [ "LATENT" ]
|
||||||
|
PARAS:
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: forward
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: [ "SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE" ]
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: [ "PROMPT" ]
|
||||||
|
|
||||||
|
MODEL:
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: FluxMR
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@flux1-dev.safetensors
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
OUT_CHANNELS: 64
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
NUM_HEADS: 24
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
THETA: 10000
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
GUIDANCE_EMBED: True
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
QKV_BIAS: True
|
||||||
|
DEPTH: 19
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
ATTN_BACKEND: flash_attn
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: False
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: False
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
T5_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder_2/
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer_2/
|
||||||
|
MAX_LENGTH: 512
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
CLIP_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder/
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer/
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
NAME: ACE_0.6B_512
|
||||||
|
IS_DEFAULT: True
|
||||||
|
USE_DYNAMIC_MODEL: True
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
#
|
||||||
|
INPUT:
|
||||||
|
INPUT_IMAGE:
|
||||||
|
INPUT_MASK:
|
||||||
|
TASK:
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
OUTPUT_HEIGHT: 512
|
||||||
|
OUTPUT_WIDTH: 512
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 20
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
SEED: -1
|
||||||
|
TAR_INDEX: 0
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
- NAME: decode
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: forward
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE"]
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
- NAME: encode_list_of_list
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionACE
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT: ""
|
||||||
|
TEXT_IDENTIFIER: [ '{image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
USE_TEXT_POS_EMBEDDINGS: True
|
||||||
|
#
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: eps
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: LinearScheduler
|
||||||
|
NUM_TIMESTEPS: 1000
|
||||||
|
BETA_MIN: 0.0001
|
||||||
|
BETA_MAX: 0.02
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: ACE
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-512px@models/dit/ace_0.6b_512px.pth
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
Y_CHANNELS: 4096
|
||||||
|
MAX_SEQ_LEN: 1024
|
||||||
|
QK_NORM: True
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
ATTENTION_BACKEND: flash_attn
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
EMBED_DIM: 4
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-512px@models/vae/vae.bin
|
||||||
|
IGNORE_KEYS: []
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://iic/ACE-0.6B-512px@models/text_encoder/t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://iic/ACE-0.6B-512px@models/tokenizer/t5-v1_1-xxl
|
||||||
|
LENGTH: 120
|
||||||
|
T5_DTYPE: bfloat16
|
||||||
|
ADDED_IDENTIFIER: [ '{image}', '{caption}', '{mask}', '{ref_image}', '{image1}', '{image2}', '{image3}', '{image4}', '{image5}', '{image6}', '{image7}', '{image8}', '{image9}' ]
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,151 @@
|
|||||||
|
NAME: COGVIDEOX_2B
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [[480, 720]]
|
||||||
|
INPUT:
|
||||||
|
IMAGE:
|
||||||
|
ORIGINAL_SIZE_AS_TUPLE: [480, 720]
|
||||||
|
TARGET_SIZE_AS_TUPLE: [480, 720]
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
SAMPLE: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
NUM_FRAMES:
|
||||||
|
DEFAULT: 49
|
||||||
|
VISIBLE: True
|
||||||
|
FPS:
|
||||||
|
DEFAULT: 8
|
||||||
|
VISIBLE: True
|
||||||
|
OUTPUT:
|
||||||
|
VIDEOS:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: decode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
PARAS:
|
||||||
|
SCALING_FACTOR_IMAGE: 1.15258426
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: forward
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION", "NUM_FRAMES", "FPS"]
|
||||||
|
PARAS:
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: False
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
LATENT_CHANNELS: 16
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 3.0
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@transformer/diffusion_pytorch_model.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 30
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 30
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: False
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: False
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@vae/diffusion_pytorch_model.safetensors
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
NAME: COGVIDEOX_5B
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [[480, 720]]
|
||||||
|
INPUT:
|
||||||
|
IMAGE:
|
||||||
|
ORIGINAL_SIZE_AS_TUPLE: [480, 720]
|
||||||
|
TARGET_SIZE_AS_TUPLE: [480, 720]
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
SAMPLE: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
NUM_FRAMES:
|
||||||
|
DEFAULT: 49
|
||||||
|
VISIBLE: True
|
||||||
|
FPS:
|
||||||
|
DEFAULT: 8
|
||||||
|
VISIBLE: True
|
||||||
|
OUTPUT:
|
||||||
|
VIDEOS:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: decode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
PARAS:
|
||||||
|
SCALING_FACTOR_IMAGE: 0.7 # 5b diff
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: forward
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION", "NUM_FRAMES", "FPS"]
|
||||||
|
PARAS:
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: True
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
LATENT_CHANNELS: 16
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 1.0 # 5b diff
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: # 5b diff
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b@transformer/diffusion_pytorch_model-00001-of-00002.safetensors
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b@transformer/diffusion_pytorch_model-00002-of-00002.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 48 # 5b diff
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 42 # 5b diff
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: True # 5b diff
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: False
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-5b@vae/diffusion_pytorch_model.safetensors # 5b diff
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
NAME: FLUX1.0_DEV
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [[1024, 1024]]
|
||||||
|
INPUT:
|
||||||
|
IMAGE:
|
||||||
|
ORIGINAL_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
TARGET_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT:
|
||||||
|
DEFAULT: ""
|
||||||
|
VISIBLE: False
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
SAMPLE:
|
||||||
|
VALUES: ["flow_euler"]
|
||||||
|
DEFAULT: "flow_euler"
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
GUIDE_RESCALE:
|
||||||
|
DEFAULT: 0.0
|
||||||
|
VISIBLE: False
|
||||||
|
DISCRETIZATION:
|
||||||
|
VALUES: []
|
||||||
|
DEFAULT:
|
||||||
|
VISIBLE: False
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
-
|
||||||
|
NAME: decode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
PARAS:
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: forward
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE"]
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionFlux
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'DiffusionFluxRF'
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
# NOISE_SCHEDULER DESCRIPTION: TYPE: default: ''
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'FlowMatchSigmaScheduler'
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
# WEIGHTING_SCHEME DESCRIPTION: The weighting scheme for sampling timesteps, choose from ['sigma_sqrt', 'logit_normal', 'mode', 'cosmap', 'none']. TYPE: str default: 'logit_normal'
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
# LOGIT_MEAN DESCRIPTION: The mean of the logit distribution for sampling timesteps. TYPE: float default: 0.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
# LOGIT_STD DESCRIPTION: The standard deviation of the logit distribution for sampling timesteps. TYPE: float default: 1.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
# MODE_SCALE DESCRIPTION: The scale factor for the mode of the logit distribution for sampling timesteps. TYPE: float default: 1.29
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'Flux'
|
||||||
|
NAME: Flux
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@flux1-dev.safetensors
|
||||||
|
# IN_CHANNELS DESCRIPTION: model's input channels. TYPE: int default: 64
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
# HIDDEN_SIZE DESCRIPTION: model's hidden size. TYPE: int default: 1024
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
# NUM_HEADS DESCRIPTION: number of heads in the transformer. TYPE: int default: 16
|
||||||
|
NUM_HEADS: 24
|
||||||
|
# AXES_DIM DESCRIPTION: dimensions of the axes of the positional encoding. TYPE: list default: [16, 56, 56]
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
# THETA DESCRIPTION: theta for positional encoding. TYPE: int default: 10000
|
||||||
|
THETA: 10000
|
||||||
|
# VEC_IN_DIM DESCRIPTION: dimension of the vector input. TYPE: int default: 768
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
# GUIDANCE_EMBED DESCRIPTION: whether to use guidance embedding. TYPE: bool default: False
|
||||||
|
GUIDANCE_EMBED: True
|
||||||
|
# CONTEXT_IN_DIM DESCRIPTION: dimension of the context input. TYPE: int default: 4096
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
# MLP_RATIO DESCRIPTION: ratio of mlp hidden size to hidden size. TYPE: float default: 4.0
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
# QKV_BIAS DESCRIPTION: whether to use bias in qkv projection. TYPE: bool default: True
|
||||||
|
QKV_BIAS: True
|
||||||
|
# DEPTH DESCRIPTION: number of transformer blocks. TYPE: int default: 19
|
||||||
|
DEPTH: 19
|
||||||
|
# DEPTH_SINGLE_BLOCKS DESCRIPTION: number of transformer blocks in the single stream block. TYPE: int default: 38
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'T5PlusClipFluxEmbedder'
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
# T5_MODEL DESCRIPTION: TYPE: default: ''
|
||||||
|
T5_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'HFEmbedder'
|
||||||
|
NAME: HFEmbedder
|
||||||
|
# HF_MODEL_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
# MODEL_PATH DESCRIPTION: model folder path TYPE: NoneType default: None
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder_2/
|
||||||
|
# HF_TOKENIZER_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
# TOKENIZER_PATH DESCRIPTION: tokenizer folder path TYPE: NoneType default: None
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer_2/
|
||||||
|
# MAX_LENGTH DESCRIPTION: max length of input TYPE: int default: 77
|
||||||
|
MAX_LENGTH: 512
|
||||||
|
# OUTPUT_KEY DESCRIPTION: output key TYPE: str default: 'last_hidden_state'
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
# D_TYPE DESCRIPTION: dtype TYPE: str default: 'bfloat16'
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
# BATCH_INFER DESCRIPTION: batch infer TYPE: bool default: False
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
# CLIP_MODEL DESCRIPTION: TYPE: default: ''
|
||||||
|
CLIP_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'HFEmbedder'
|
||||||
|
NAME: HFEmbedder
|
||||||
|
# HF_MODEL_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
# MODEL_PATH DESCRIPTION: model folder path TYPE: NoneType default: None
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder/
|
||||||
|
# HF_TOKENIZER_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
# TOKENIZER_PATH DESCRIPTION: tokenizer folder path TYPE: NoneType default: None
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer/
|
||||||
|
# MAX_LENGTH DESCRIPTION: max length of input TYPE: int default: 77
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
# OUTPUT_KEY DESCRIPTION: output key TYPE: str default: 'last_hidden_state'
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
# D_TYPE DESCRIPTION: dtype TYPE: str default: 'bfloat16'
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
# BATCH_INFER DESCRIPTION: batch infer TYPE: bool default: False
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
NAME: FLUX1.0_SCHNELL
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [[1024, 1024]]
|
||||||
|
INPUT:
|
||||||
|
IMAGE:
|
||||||
|
ORIGINAL_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
TARGET_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT:
|
||||||
|
DEFAULT: ""
|
||||||
|
VISIBLE: False
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
SAMPLE:
|
||||||
|
VALUES: ["flow_euler"]
|
||||||
|
DEFAULT: "flow_euler"
|
||||||
|
SAMPLE_STEPS: 4
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
GUIDE_RESCALE:
|
||||||
|
DEFAULT: 0.0
|
||||||
|
VISIBLE: False
|
||||||
|
DISCRETIZATION:
|
||||||
|
VALUES: []
|
||||||
|
DEFAULT:
|
||||||
|
VISIBLE: False
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
-
|
||||||
|
NAME: decode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
PARAS:
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: forward
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE"]
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: bfloat16
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionFlux
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'DiffusionFluxRF'
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
# NOISE_SCHEDULER DESCRIPTION: TYPE: default: ''
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'FlowMatchSigmaScheduler'
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
# WEIGHTING_SCHEME DESCRIPTION: The weighting scheme for sampling timesteps, choose from ['sigma_sqrt', 'logit_normal', 'mode', 'cosmap', 'none']. TYPE: str default: 'logit_normal'
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
# LOGIT_MEAN DESCRIPTION: The mean of the logit distribution for sampling timesteps. TYPE: float default: 0.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
# LOGIT_STD DESCRIPTION: The standard deviation of the logit distribution for sampling timesteps. TYPE: float default: 1.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
# MODE_SCALE DESCRIPTION: The scale factor for the mode of the logit distribution for sampling timesteps. TYPE: float default: 1.29
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
SAMPLER_SCHEDULER:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'FlowMatchFluxShiftScheduler'
|
||||||
|
NAME: FlowMatchFluxShiftScheduler
|
||||||
|
# SHIFT DESCRIPTION: Use timestamp shift or not, default is True. TYPE: bool default: True
|
||||||
|
SHIFT: False
|
||||||
|
# SIGMOID_SCALE DESCRIPTION: The scale of sigmoid function for sampling timesteps. TYPE: int default: 1
|
||||||
|
SIGMOID_SCALE: 1
|
||||||
|
# BASE_SHIFT DESCRIPTION: The base shift factor for the timestamp. TYPE: float default: 0.5
|
||||||
|
BASE_SHIFT: 0.5
|
||||||
|
# MAX_SHIFT DESCRIPTION: The max shift factor for the timestamp. TYPE: float default: 1.15
|
||||||
|
MAX_SHIFT: 1.15
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'Flux'
|
||||||
|
NAME: Flux
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-schnell@flux1-schnell.safetensors
|
||||||
|
# IN_CHANNELS DESCRIPTION: model's input channels. TYPE: int default: 64
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
# HIDDEN_SIZE DESCRIPTION: model's hidden size. TYPE: int default: 1024
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
# NUM_HEADS DESCRIPTION: number of heads in the transformer. TYPE: int default: 16
|
||||||
|
NUM_HEADS: 24
|
||||||
|
# AXES_DIM DESCRIPTION: dimensions of the axes of the positional encoding. TYPE: list default: [16, 56, 56]
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
# THETA DESCRIPTION: theta for positional encoding. TYPE: int default: 10000
|
||||||
|
THETA: 10000
|
||||||
|
# VEC_IN_DIM DESCRIPTION: dimension of the vector input. TYPE: int default: 768
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
# GUIDANCE_EMBED DESCRIPTION: whether to use guidance embedding. TYPE: bool default: False
|
||||||
|
GUIDANCE_EMBED: False
|
||||||
|
# CONTEXT_IN_DIM DESCRIPTION: dimension of the context input. TYPE: int default: 4096
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
# MLP_RATIO DESCRIPTION: ratio of mlp hidden size to hidden size. TYPE: float default: 4.0
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
# QKV_BIAS DESCRIPTION: whether to use bias in qkv projection. TYPE: bool default: True
|
||||||
|
QKV_BIAS: True
|
||||||
|
# DEPTH DESCRIPTION: number of transformer blocks. TYPE: int default: 19
|
||||||
|
DEPTH: 19
|
||||||
|
# DEPTH_SINGLE_BLOCKS DESCRIPTION: number of transformer blocks in the single stream block. TYPE: int default: 38
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-schnell@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'T5PlusClipFluxEmbedder'
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
# T5_MODEL DESCRIPTION: TYPE: default: ''
|
||||||
|
T5_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'HFEmbedder'
|
||||||
|
NAME: HFEmbedder
|
||||||
|
# HF_MODEL_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
# MODEL_PATH DESCRIPTION: model folder path TYPE: NoneType default: None
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-schnell@text_encoder_2/
|
||||||
|
# HF_TOKENIZER_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
# TOKENIZER_PATH DESCRIPTION: tokenizer folder path TYPE: NoneType default: None
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-schnell@tokenizer_2/
|
||||||
|
# MAX_LENGTH DESCRIPTION: max length of input TYPE: int default: 77
|
||||||
|
MAX_LENGTH: 256
|
||||||
|
# OUTPUT_KEY DESCRIPTION: output key TYPE: str default: 'last_hidden_state'
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
# D_TYPE DESCRIPTION: dtype TYPE: str default: 'bfloat16'
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
# BATCH_INFER DESCRIPTION: batch infer TYPE: bool default: False
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
# CLIP_MODEL DESCRIPTION: TYPE: default: ''
|
||||||
|
CLIP_MODEL:
|
||||||
|
# NAME DESCRIPTION: TYPE: default: 'HFEmbedder'
|
||||||
|
NAME: HFEmbedder
|
||||||
|
# HF_MODEL_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
# MODEL_PATH DESCRIPTION: model folder path TYPE: NoneType default: None
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-schnell@text_encoder/
|
||||||
|
# HF_TOKENIZER_CLS DESCRIPTION: huggingface cls in transfomer TYPE: NoneType default: None
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
# TOKENIZER_PATH DESCRIPTION: tokenizer folder path TYPE: NoneType default: None
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-schnell@tokenizer/
|
||||||
|
# MAX_LENGTH DESCRIPTION: max length of input TYPE: int default: 77
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
# OUTPUT_KEY DESCRIPTION: output key TYPE: str default: 'last_hidden_state'
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
# D_TYPE DESCRIPTION: dtype TYPE: str default: 'bfloat16'
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
# BATCH_INFER DESCRIPTION: batch infer TYPE: bool default: False
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
NAME: PIXART_ALPHA
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [[1024, 1024]]
|
||||||
|
INPUT:
|
||||||
|
IMAGE:
|
||||||
|
ORIGINAL_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
TARGET_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
SAMPLE: ddim
|
||||||
|
SAMPLE_STEPS: 20
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: float32
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
-
|
||||||
|
NAME: decode
|
||||||
|
DTYPE: float32
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
PARAS:
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: forward
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION"]
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: float32
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
SCHEDULE:
|
||||||
|
PARAMETERIZATION: "eps"
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "linear"
|
||||||
|
"BETA_MIN": 0.0001
|
||||||
|
"BETA_MAX": 0.02
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: PixArt
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@PixArt-XL-2-1024-MS.pth
|
||||||
|
INPUT_SIZE: 128
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
CLASS_DROPOUT_PROB: 0.1
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
USE_REL_POS: False
|
||||||
|
CAPTION_CHANNELS: 4096
|
||||||
|
LEWEI_SCALE: 2
|
||||||
|
MODEL_MAX_LENGTH: 120
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-2-base@512-base-ema.safetensors
|
||||||
|
EMBED_DIM: 4
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/
|
||||||
|
LENGTH: 120
|
||||||
|
CLEAN: heavy
|
||||||
|
USE_GRAD: False
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
NAME: SD3
|
||||||
|
IS_DEFAULT: False
|
||||||
|
DEFAULT_PARAS:
|
||||||
|
PARAS:
|
||||||
|
RESOLUTIONS: [[1024, 1024]]
|
||||||
|
INPUT:
|
||||||
|
IMAGE:
|
||||||
|
ORIGINAL_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
TARGET_SIZE_AS_TUPLE: [1024, 1024]
|
||||||
|
PROMPT: ""
|
||||||
|
NEGATIVE_PROMPT: ""
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
SAMPLE: euler
|
||||||
|
SAMPLE_STEPS: 28
|
||||||
|
GUIDE_SCALE: 5.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
OUTPUT:
|
||||||
|
LATENT:
|
||||||
|
IMAGES:
|
||||||
|
SEED:
|
||||||
|
MODULES_PARAS:
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: float32
|
||||||
|
INPUT: ["IMAGE"]
|
||||||
|
-
|
||||||
|
NAME: decode
|
||||||
|
DTYPE: float32
|
||||||
|
INPUT: ["LATENT"]
|
||||||
|
PARAS:
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: forward
|
||||||
|
DTYPE: float16
|
||||||
|
INPUT: ["SAMPLE_STEPS", "SAMPLE", "GUIDE_SCALE", "GUIDE_RESCALE", "DISCRETIZATION"]
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
FUNCTION:
|
||||||
|
-
|
||||||
|
NAME: encode
|
||||||
|
DTYPE: float32
|
||||||
|
INPUT: ["PROMPT"]
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
SCHEDULE:
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "shifted"
|
||||||
|
"SHIFT": 3
|
||||||
|
T_WEIGHT: uniform
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: MMDiT
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
IGNORE_KEYS: '^first_stage_model.'
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
DEPTH: 24
|
||||||
|
INPUT_SIZE:
|
||||||
|
ADM_IN_CHANNELS: 2048
|
||||||
|
CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } }
|
||||||
|
NUM_PATCHES: 36864
|
||||||
|
POS_EMBED_MAX_SIZE: 192
|
||||||
|
POS_EMBED_SCALING_FACTOR:
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
EMBED_DIM: 16
|
||||||
|
IGNORE_KEYS: '^model.diffusion_model.'
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
USE_CONV: False
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: SD3TextEmbedder
|
||||||
|
P_ZERO: 0.0
|
||||||
|
CLIP_L:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
CLIP_G:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
T5_XXL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3
|
||||||
|
LENGTH: 256
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
T5_DTYPE: float16
|
||||||
@@ -6,65 +6,91 @@ DIFFUSION_PARAS:
|
|||||||
'dpm2_karras', 'dpm2_ancestral_karras', 'dpmpp_2s_ancestral_karras', 'dpmpp_2m_karras',
|
'dpm2_karras', 'dpm2_ancestral_karras', 'dpmpp_2s_ancestral_karras', 'dpmpp_2m_karras',
|
||||||
'dpmpp_sde_karras', 'dpmpp_2m_sde_karras']
|
'dpmpp_sde_karras', 'dpmpp_2m_sde_karras']
|
||||||
DEFAULT: 'dpmpp_2s_ancestral'
|
DEFAULT: 'dpmpp_2s_ancestral'
|
||||||
|
VISIBLE: True
|
||||||
NEGATIVE_PROMPT:
|
NEGATIVE_PROMPT:
|
||||||
DEFAULT:
|
DEFAULT:
|
||||||
|
VISIBLE: True
|
||||||
PROMPT_PREFIX:
|
PROMPT_PREFIX:
|
||||||
DEFAULT:
|
DEFAULT:
|
||||||
|
VISIBLE: True
|
||||||
SAMPLES:
|
SAMPLES:
|
||||||
MIN: 1
|
MIN: 1
|
||||||
MAX: 4
|
MAX: 4
|
||||||
DEFAULT: 1
|
DEFAULT: 1
|
||||||
|
VISIBLE: True
|
||||||
|
NUM_FRAMES:
|
||||||
|
MIN: 1
|
||||||
|
MAX: 100
|
||||||
|
DEFAULT: 49
|
||||||
|
VISIBLE: False
|
||||||
|
FPS:
|
||||||
|
MIN: 1
|
||||||
|
MAX: 50
|
||||||
|
DEFAULT: 8
|
||||||
|
VISIBLE: False
|
||||||
SAMPLE_STEPS:
|
SAMPLE_STEPS:
|
||||||
MIN: 1
|
MIN: 1
|
||||||
MAX: 100
|
MAX: 100
|
||||||
DEFAULT: 30
|
DEFAULT: 30
|
||||||
|
VISIBLE: True
|
||||||
GUIDE_SCALE:
|
GUIDE_SCALE:
|
||||||
MIN: 0
|
MIN: 0
|
||||||
MAX: 10
|
MAX: 10
|
||||||
DEFAULT: 5.0
|
DEFAULT: 5.0
|
||||||
|
VISIBLE: True
|
||||||
GUIDE_RESCALE:
|
GUIDE_RESCALE:
|
||||||
MIN: 0
|
MIN: 0
|
||||||
MAX: 1.0
|
MAX: 1.0
|
||||||
DEFAULT: 0.5
|
DEFAULT: 0.5
|
||||||
|
VISIBLE: True
|
||||||
DISCRETIZATION:
|
DISCRETIZATION:
|
||||||
VALUES: ["trailing", "leading", "linspace"]
|
VALUES: ["trailing", "leading", "linspace"]
|
||||||
DEFAULT: "linspace"
|
DEFAULT: "linspace"
|
||||||
|
VISIBLE: True
|
||||||
REFINE_SAMPLERS:
|
REFINE_SAMPLERS:
|
||||||
VALUES: [ 'ddim', 'euler', 'euler_ancestral', 'heun', 'dpm2',
|
VALUES: [ 'ddim', 'euler', 'euler_ancestral', 'heun', 'dpm2',
|
||||||
'dpm2_ancestral', 'dpmpp_2m', 'dpmpp_sde', 'dpmpp_2m_sde', 'dpmpp_2s_ancestral',
|
'dpm2_ancestral', 'dpmpp_2m', 'dpmpp_sde', 'dpmpp_2m_sde', 'dpmpp_2s_ancestral',
|
||||||
'dpm2_karras', 'dpm2_ancestral_karras', 'dpmpp_2s_ancestral_karras', 'dpmpp_2m_karras',
|
'dpm2_karras', 'dpm2_ancestral_karras', 'dpmpp_2s_ancestral_karras', 'dpmpp_2m_karras',
|
||||||
'dpmpp_sde_karras', 'dpmpp_2m_sde_karras' ]
|
'dpmpp_sde_karras', 'dpmpp_2m_sde_karras' ]
|
||||||
DEFAULT: 'dpmpp_2s_ancestral'
|
DEFAULT: 'dpmpp_2s_ancestral'
|
||||||
|
VISIBLE: True
|
||||||
REFINE_SAMPLE_STEPS:
|
REFINE_SAMPLE_STEPS:
|
||||||
MIN: 0
|
MIN: 0
|
||||||
MAX: 100
|
MAX: 100
|
||||||
DEFAULT: 30
|
DEFAULT: 30
|
||||||
|
VISIBLE: True
|
||||||
REFINE_GUIDE_SCALE:
|
REFINE_GUIDE_SCALE:
|
||||||
MIN: 0
|
MIN: 0
|
||||||
MAX: 10
|
MAX: 10
|
||||||
DEFAULT: 5.0
|
DEFAULT: 5.0
|
||||||
|
VISIBLE: True
|
||||||
REFINE_GUIDE_RESCALE:
|
REFINE_GUIDE_RESCALE:
|
||||||
MIN: 0
|
MIN: 0
|
||||||
MAX: 1.0
|
MAX: 1.0
|
||||||
DEFAULT: 0.5
|
DEFAULT: 0.5
|
||||||
|
VISIBLE: True
|
||||||
REFINE_DISCRETIZATION:
|
REFINE_DISCRETIZATION:
|
||||||
VALUES: [ "trailing", "leading", "linspace" ]
|
VALUES: [ "trailing", "leading", "linspace" ]
|
||||||
DEFAULT: "linspace"
|
DEFAULT: "linspace"
|
||||||
|
VISIBLE: True
|
||||||
AESTHETIC_SCORE:
|
AESTHETIC_SCORE:
|
||||||
MIN: 0.0
|
MIN: 0.0
|
||||||
MAX: 10.0
|
MAX: 10.0
|
||||||
DEFAULT: 6.0
|
DEFAULT: 6.0
|
||||||
|
VISIBLE: True
|
||||||
NEGATIVE_AESTHETIC_SCORE:
|
NEGATIVE_AESTHETIC_SCORE:
|
||||||
MIN: 0.0
|
MIN: 0.0
|
||||||
MAX: 10.0
|
MAX: 10.0
|
||||||
DEFAULT: 2.5
|
DEFAULT: 2.5
|
||||||
|
VISIBLE: True
|
||||||
REFINE_STRENGTH:
|
REFINE_STRENGTH:
|
||||||
MIN: 0
|
MIN: 0
|
||||||
MAX: 1.0
|
MAX: 1.0
|
||||||
DEFAULT: 0.15
|
DEFAULT: 0.15
|
||||||
RESOLUTIONS:
|
RESOLUTIONS:
|
||||||
VALUES: [ [512, 512], [768, 768],
|
VALUES: [
|
||||||
[704, 1408], [704, 1344], [768, 1344],
|
[512, 512], [768, 768],
|
||||||
|
[704, 1408], [704, 1344], [768, 1344],
|
||||||
[720, 1280],
|
[720, 1280],
|
||||||
[768, 1280], [832, 1216], [832, 1152],
|
[768, 1280], [832, 1216], [832, 1152],
|
||||||
[896, 1152], [896, 1088], [960, 1088],
|
[896, 1152], [896, 1088], [960, 1088],
|
||||||
@@ -74,8 +100,14 @@ DIFFUSION_PARAS:
|
|||||||
[1280, 768],
|
[1280, 768],
|
||||||
[1344, 768], [1344, 704], [1408, 704],
|
[1344, 768], [1344, 704], [1408, 704],
|
||||||
[1472, 704], [1536, 640], [1600, 640],
|
[1472, 704], [1536, 640], [1600, 640],
|
||||||
[1664, 576], [1728, 576]]
|
[1664, 576], [1728, 576],
|
||||||
|
[2048, 2048], [2048, 1920], [1920, 2048],
|
||||||
|
[1536, 2560], [2560, 1536], [2560, 1440],
|
||||||
|
[2560, 1440],
|
||||||
|
[480, 720], [720, 480]
|
||||||
|
]
|
||||||
DEFAULT: [1024, 1024]
|
DEFAULT: [1024, 1024]
|
||||||
|
VISIBLE: True
|
||||||
EXTENSION_PARAS:
|
EXTENSION_PARAS:
|
||||||
MANTRA_BOOK: scepter/methods/studio/extensions/mantra_book/mantra_book.yaml
|
MANTRA_BOOK: scepter/methods/studio/extensions/mantra_book/mantra_book.yaml
|
||||||
OFFICIAL_TUNERS: scepter/methods/studio/extensions/tuners/official_tuners.yaml
|
OFFICIAL_TUNERS: scepter/methods/studio/extensions/tuners/official_tuners.yaml
|
||||||
|
|||||||
@@ -2,11 +2,10 @@ WORK_DIR: datasets
|
|||||||
EXPORT_DIR: export_datasets
|
EXPORT_DIR: export_datasets
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
-
|
-
|
||||||
# NAME DESCRIPTION: TYPE: default: ''
|
|
||||||
NAME: LocalFs
|
NAME: LocalFs
|
||||||
AUTO_CLEAN: False
|
AUTO_CLEAN: False
|
||||||
|
|
||||||
PROCESSORS:
|
PROCESSORS:
|
||||||
|
# Caption processor
|
||||||
- NAME: BlipImageBase
|
- NAME: BlipImageBase
|
||||||
TYPE: caption
|
TYPE: caption
|
||||||
MODEL_PATH: ms://cubeai/blip-image-captioning-base
|
MODEL_PATH: ms://cubeai/blip-image-captioning-base
|
||||||
@@ -15,6 +14,81 @@ PROCESSORS:
|
|||||||
PARAS:
|
PARAS:
|
||||||
- LANGUAGE_NAME: English
|
- LANGUAGE_NAME: English
|
||||||
LANGUAGE_ZH_NAME: 英语
|
LANGUAGE_ZH_NAME: 英语
|
||||||
|
- NAME: InternVL15
|
||||||
|
TYPE: caption
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/InternVL-Chat-V1-5
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 49968
|
||||||
|
PARAS:
|
||||||
|
- PROMPT: 用中文描述这张图片
|
||||||
|
LANGUAGE_NAME: Chinese
|
||||||
|
LANGUAGE_ZH_NAME: 中文
|
||||||
|
- PROMPT: Generate the caption in English
|
||||||
|
LANGUAGE_NAME: English
|
||||||
|
LANGUAGE_ZH_NAME: 英语
|
||||||
|
- NAME: QWVLQuantize
|
||||||
|
TYPE: caption
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 7885
|
||||||
|
MODEL_PATH: ms://qwen/Qwen-VL:v1.0.3
|
||||||
|
PARAS:
|
||||||
|
- PROMPT: 用中文描述这张图片
|
||||||
|
LANGUAGE_NAME: Chinese
|
||||||
|
LANGUAGE_ZH_NAME: 中文
|
||||||
|
MAX_NEW_TOKENS:
|
||||||
|
VALUE: 1024
|
||||||
|
MAX: 2048
|
||||||
|
STEP: 128
|
||||||
|
MIN: 256
|
||||||
|
MIN_NEW_TOKENS:
|
||||||
|
VALUE: 16
|
||||||
|
MAX: 1024
|
||||||
|
STEP: 16
|
||||||
|
MIN: 0
|
||||||
|
NUM_BEAMS:
|
||||||
|
VALUE: 1
|
||||||
|
MAX: 12
|
||||||
|
STEP: 1
|
||||||
|
MIN: 1
|
||||||
|
REPETITION_PENALTY:
|
||||||
|
VALUE: 1.0
|
||||||
|
MAX: 100.0
|
||||||
|
STEP: 1.0
|
||||||
|
MIN: 1.0
|
||||||
|
TEMPERATURE:
|
||||||
|
VALUE: 1.0
|
||||||
|
MAX: 100.0
|
||||||
|
STEP: 1.0
|
||||||
|
MIN: 1.0
|
||||||
|
- PROMPT: Generate the caption in English
|
||||||
|
LANGUAGE_NAME: English
|
||||||
|
LANGUAGE_ZH_NAME: 英语
|
||||||
|
MAX_NEW_TOKENS:
|
||||||
|
VALUE: 1024
|
||||||
|
MAX: 2048
|
||||||
|
STEP: 128
|
||||||
|
MIN: 256
|
||||||
|
MIN_NEW_TOKENS:
|
||||||
|
VALUE: 16
|
||||||
|
MAX: 1024
|
||||||
|
STEP: 16
|
||||||
|
MIN: 0
|
||||||
|
NUM_BEAMS:
|
||||||
|
VALUE: 1
|
||||||
|
MAX: 12
|
||||||
|
STEP: 1
|
||||||
|
MIN: 1
|
||||||
|
REPETITION_PENALTY:
|
||||||
|
VALUE: 1.0
|
||||||
|
MAX: 100.0
|
||||||
|
STEP: 1.0
|
||||||
|
MIN: 1.0
|
||||||
|
TEMPERATURE:
|
||||||
|
VALUE: 1.0
|
||||||
|
MAX: 100.0
|
||||||
|
STEP: 1.0
|
||||||
|
MIN: 1.0
|
||||||
|
|
||||||
- NAME: QWVL
|
- NAME: QWVL
|
||||||
TYPE: caption
|
TYPE: caption
|
||||||
MODEL_PATH: ms://qwen/Qwen-VL:v1.0.3
|
MODEL_PATH: ms://qwen/Qwen-VL:v1.0.3
|
||||||
@@ -77,75 +151,14 @@ PROCESSORS:
|
|||||||
MAX: 100.0
|
MAX: 100.0
|
||||||
STEP: 1.0
|
STEP: 1.0
|
||||||
MIN: 1.0
|
MIN: 1.0
|
||||||
-
|
|
||||||
NAME: QWVLQuantize
|
# Simple processor
|
||||||
TYPE: caption
|
- NAME: CenterCrop
|
||||||
DEVICE: "gpu"
|
|
||||||
MEMORY: 7885
|
|
||||||
MODEL_PATH: ms://qwen/Qwen-VL:v1.0.3
|
|
||||||
PARAS:
|
|
||||||
- PROMPT: 用中文描述这张图片
|
|
||||||
LANGUAGE_NAME: Chinese
|
|
||||||
LANGUAGE_ZH_NAME: 中文
|
|
||||||
MAX_NEW_TOKENS:
|
|
||||||
VALUE: 1024
|
|
||||||
MAX: 2048
|
|
||||||
STEP: 128
|
|
||||||
MIN: 256
|
|
||||||
MIN_NEW_TOKENS:
|
|
||||||
VALUE: 16
|
|
||||||
MAX: 1024
|
|
||||||
STEP: 16
|
|
||||||
MIN: 0
|
|
||||||
NUM_BEAMS:
|
|
||||||
VALUE: 1
|
|
||||||
MAX: 12
|
|
||||||
STEP: 1
|
|
||||||
MIN: 1
|
|
||||||
REPETITION_PENALTY:
|
|
||||||
VALUE: 1.0
|
|
||||||
MAX: 100.0
|
|
||||||
STEP: 1.0
|
|
||||||
MIN: 1.0
|
|
||||||
TEMPERATURE:
|
|
||||||
VALUE: 1.0
|
|
||||||
MAX: 100.0
|
|
||||||
STEP: 1.0
|
|
||||||
MIN: 1.0
|
|
||||||
- PROMPT: Generate the caption in English
|
|
||||||
LANGUAGE_NAME: English
|
|
||||||
LANGUAGE_ZH_NAME: 英语
|
|
||||||
MAX_NEW_TOKENS:
|
|
||||||
VALUE: 1024
|
|
||||||
MAX: 2048
|
|
||||||
STEP: 128
|
|
||||||
MIN: 256
|
|
||||||
MIN_NEW_TOKENS:
|
|
||||||
VALUE: 16
|
|
||||||
MAX: 1024
|
|
||||||
STEP: 16
|
|
||||||
MIN: 0
|
|
||||||
NUM_BEAMS:
|
|
||||||
VALUE: 1
|
|
||||||
MAX: 12
|
|
||||||
STEP: 1
|
|
||||||
MIN: 1
|
|
||||||
REPETITION_PENALTY:
|
|
||||||
VALUE: 1.0
|
|
||||||
MAX: 100.0
|
|
||||||
STEP: 1.0
|
|
||||||
MIN: 1.0
|
|
||||||
TEMPERATURE:
|
|
||||||
VALUE: 1.0
|
|
||||||
MAX: 100.0
|
|
||||||
STEP: 1.0
|
|
||||||
MIN: 1.0
|
|
||||||
-
|
|
||||||
NAME: CenterCrop
|
|
||||||
TYPE: image
|
TYPE: image
|
||||||
DEVICE: "cpu"
|
DEVICE: "cpu"
|
||||||
MEMORY: 10
|
MEMORY: 10
|
||||||
PARAS:
|
PARAS:
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
HEIGHT_RATIO:
|
HEIGHT_RATIO:
|
||||||
VALUE: 1
|
VALUE: 1
|
||||||
MAX: 20
|
MAX: 20
|
||||||
@@ -156,18 +169,309 @@ PROCESSORS:
|
|||||||
MAX: 20
|
MAX: 20
|
||||||
STEP: 1
|
STEP: 1
|
||||||
MIN: 1
|
MIN: 1
|
||||||
# - NAME: PaddingCrop
|
- NAME: ChangeSample
|
||||||
# TYPE: image
|
TYPE: image
|
||||||
# DEVICE: "cpu"
|
DEVICE: "cpu"
|
||||||
# MEMORY: 10
|
MEMORY: 10
|
||||||
# PARAS:
|
PARAS:
|
||||||
# HEIGHT_RATIO:
|
CAPTION_INTERACTIVE: False
|
||||||
# VALUE: 3
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
# MAX: 25
|
SRC_IMAGE_MASK_INTERACTIVE: True
|
||||||
# STEP: 1
|
TARGET_IMAGE_INTERACTIVE: True
|
||||||
# MIN: 1
|
PREVIEW_BTN_VISIBLE: False
|
||||||
# WIDTH_RATIO:
|
- NAME: MaskEditSample
|
||||||
# VALUE: 4
|
TYPE: image
|
||||||
# MAX: 20
|
DEVICE: "cpu"
|
||||||
# STEP: 1
|
MEMORY: 10
|
||||||
# MIN: 1
|
PARAS:
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: SourceMaskSample
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: MaskSwapEditSample
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
PARAS:
|
||||||
|
PREVIEW_BTN_VISIBLE: False
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: SwapMaskSwapEditSample
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
PARAS:
|
||||||
|
PREVIEW_BTN_VISIBLE: False
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: SwapSample
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
PARAS:
|
||||||
|
PREVIEW_BTN_VISIBLE: True
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
TARGET_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: DefaultMaskSample
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
PARAS:
|
||||||
|
PREVIEW_BTN_VISIBLE: True
|
||||||
|
SRC_IMAGE_INTERACTIVE: False
|
||||||
|
TARGET_IMAGE_INTERACTIVE: False
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
# - NAME: PaddingCrop
|
||||||
|
# TYPE: image
|
||||||
|
# DEVICE: "cpu"
|
||||||
|
# MEMORY: 10
|
||||||
|
# PARAS:
|
||||||
|
# HEIGHT_RATIO:
|
||||||
|
# VALUE: 3
|
||||||
|
# MAX: 25
|
||||||
|
# STEP: 1
|
||||||
|
# MIN: 1
|
||||||
|
# WIDTH_RATIO:
|
||||||
|
# VALUE: 4
|
||||||
|
# MAX: 20
|
||||||
|
# STEP: 1
|
||||||
|
# MIN: 1
|
||||||
|
# Annotator processor
|
||||||
|
- NAME: CannyExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "CannyAnnotator"
|
||||||
|
LOW_THRESHOLD: 100
|
||||||
|
HIGH_THRESHOLD: 200
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: ColorExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "ColorAnnotator"
|
||||||
|
RATIO: 64
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: InfoDrawContourExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "InfoDrawContourAnnotator"
|
||||||
|
INPUT_NC: 3
|
||||||
|
OUTPUT_NC: 1
|
||||||
|
N_RESIDUAL_BLOCKS: 3
|
||||||
|
SIGMOID: True
|
||||||
|
PRETRAINED_MODEL: "ms://iic/scepter_annotator@annotator/ckpts/informative_drawing_contour_style.pth"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: DegradationExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "DegradationAnnotator"
|
||||||
|
RANDOM_DEGRADATION: True
|
||||||
|
PARAMS:
|
||||||
|
gaussian_noise: { }
|
||||||
|
resize: { 'scale': [ 0.4, 0.8 ] }
|
||||||
|
jpeg: { 'jpeg_level': [ 25, 75 ] }
|
||||||
|
gaussian_blur: { 'kernel_size': [ 7, 9, 11, 13, 15 ], 'sigma': [ 0.9, 1.8 ] }
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: MidasExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "MidasDetector"
|
||||||
|
PRETRAINED_MODEL: "ms://iic/scepter_annotator@annotator/ckpts/dpt_hybrid-midas-501f0c75.pt"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: DoodleExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "DoodleAnnotator"
|
||||||
|
PROCESSOR_TYPE: "pidinet_sketch"
|
||||||
|
PROCESSOR_CFG:
|
||||||
|
- NAME: "PiDiAnnotator"
|
||||||
|
PRETRAINED_MODEL: "ms://iic/scepter_annotator@annotator/ckpts/table5_pidinet.pth"
|
||||||
|
- NAME: "SketchAnnotator"
|
||||||
|
PRETRAINED_MODEL: "ms://iic/scepter_annotator@annotator/ckpts/sketch_simplification_gan.pth"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: GrayExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "GrayAnnotator"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: InpaintingExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "InpaintingAnnotator"
|
||||||
|
RETURN_MASK: False
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
|
||||||
|
- NAME: InpaintingSourceExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: OpenposeExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "OpenposeAnnotator"
|
||||||
|
BODY_MODEL_PATH: "ms://iic/scepter_annotator@annotator/ckpts/body_pose_model.pth"
|
||||||
|
HAND_MODEL_PATH: "ms://iic/scepter_annotator@annotator/ckpts/hand_pose_model.pth"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: OutpaintingExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "OutpaintingAnnotator"
|
||||||
|
RETURN_MASK: False
|
||||||
|
KEEP_PADDING_RATIO: 1
|
||||||
|
RANDOM_CFG:
|
||||||
|
DIRECTION_RANGE: [ 'left', 'right', 'up', 'down' ]
|
||||||
|
RATIO_RANGE: [ 0.1, 0.7 ]
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
USE_MASK_VISIBLE: True
|
||||||
|
PREVIEW_BTN_VISIBLE: False
|
||||||
|
- NAME: OutpaintingResize
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "OutpaintingResize"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
PREVIEW_BTN_VISIBLE: False
|
||||||
|
- NAME: InfoDrawAnimeAnnotator
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "InfoDrawAnimeAnnotator"
|
||||||
|
INPUT_NC: 3
|
||||||
|
OUTPUT_NC: 1
|
||||||
|
N_RESIDUAL_BLOCKS: 3
|
||||||
|
SIGMOID: True
|
||||||
|
PRETRAINED_MODEL: "ms://iic/scepter_annotator@annotator/ckpts/informative_drawing_anime_style.pth"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: ESAMExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "ESAMAnnotator"
|
||||||
|
PRETRAINED_MODEL: "ms://iic/scepter_annotator@annotator/ckpts/efficient_sam_vits.pt"
|
||||||
|
SAVE_MODE: 'P'
|
||||||
|
GRID_SIZE: 32
|
||||||
|
USE_DOMINANT_COLOR: True
|
||||||
|
RETURN_MASK: False
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
- NAME: InvertExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "InvertAnnotator"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
|
||||||
|
- NAME: LamaExtractor
|
||||||
|
TYPE: image
|
||||||
|
DEVICE: "cpu"
|
||||||
|
MEMORY: 10
|
||||||
|
MODEL:
|
||||||
|
NAME: "LamaAnnotator"
|
||||||
|
PRETRAINED_MODEL: "ms:///iic/cv_fft_inpainting_lama/"
|
||||||
|
PARAS:
|
||||||
|
SRC_IMAGE_TOOL: sketch
|
||||||
|
SRC_IMAGE_INTERACTIVE: True
|
||||||
|
CAPTION_INTERACTIVE: False
|
||||||
|
|
||||||
|
VIDEO_PROCESSORS:
|
||||||
|
- NAME: CogVLM2Llama3Caption
|
||||||
|
TYPE: caption
|
||||||
|
MODEL_PATH: ms://ZhipuAI/cogvlm2-llama3-caption
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 20000
|
||||||
|
PROMPT: Please describe this video in detail.
|
||||||
|
TEMPERATURE: 0.1
|
||||||
|
MAX_NEW_TOKENS: 2048
|
||||||
|
PAD_TOKEN_ID: 128002
|
||||||
|
TOP_K: 1
|
||||||
|
TOP_P: 0.1
|
||||||
|
|
||||||
|
TRANSLATION_PROCESSORS:
|
||||||
|
- NAME: OpusMtZhEn
|
||||||
|
TYPE: caption
|
||||||
|
MODEL_PATH: ms://cubeai/trans-opus-mt-zh-en
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 5000
|
||||||
|
- NAME: OpusMtEnZh
|
||||||
|
TYPE: caption
|
||||||
|
MODEL_PATH: ms://cubeai/trans-opus-mt-en-zh
|
||||||
|
DEVICE: "gpu"
|
||||||
|
MEMORY: 5000
|
||||||
@@ -87,3 +87,7 @@ INTERFACE:
|
|||||||
NAME_EN: Inference
|
NAME_EN: Inference
|
||||||
IFID: inference
|
IFID: inference
|
||||||
CONFIG: scepter/methods/studio/inference/inference.yaml
|
CONFIG: scepter/methods/studio/inference/inference.yaml
|
||||||
|
- NAME: 对话式编辑
|
||||||
|
NAME_EN: ChatBot
|
||||||
|
IFID: chatbot
|
||||||
|
CONFIG: scepter/methods/studio/chatbot/chatbot.yaml
|
||||||
|
|||||||
@@ -0,0 +1,315 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 42
|
||||||
|
TENSOR_PARALLEL_SIZE: 1
|
||||||
|
PIPELINE_PARALLEL_SIZE: 1
|
||||||
|
SYS_ENVS:
|
||||||
|
TORCH_CUDNN_V8_API_ENABLED: '1'
|
||||||
|
TOKENIZERS_PARALLELISM: 'false'
|
||||||
|
TF_CPP_MIN_LOG_LEVEL: '3'
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
|
||||||
|
META:
|
||||||
|
VERSION: 'COGVIDEOX_2B'
|
||||||
|
DESCRIPTION: "cogvideox 2b"
|
||||||
|
IS_DEFAULT: False
|
||||||
|
IS_SHARE: True
|
||||||
|
INFERENCE_PARAS:
|
||||||
|
INFERENCE_BATCH_SIZE: 1
|
||||||
|
INFERENCE_PREFIX: ""
|
||||||
|
DEFAULT_SAMPLER: "ddim"
|
||||||
|
DEFAULT_SAMPLE_STEPS: 50
|
||||||
|
INFERENCE_N_PROMPT: ""
|
||||||
|
RESOLUTION: [ 480, 720 ]
|
||||||
|
PARAS:
|
||||||
|
- TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [ 480, 720 ]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: False
|
||||||
|
TUNER: FULL
|
||||||
|
- TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [ 480, 720 ]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: True
|
||||||
|
TUNER: LORA
|
||||||
|
#
|
||||||
|
TUNERS:
|
||||||
|
LORA:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 64
|
||||||
|
LORA_ALPHA: 64
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.to_k|.to_q|.to_v|.to_out.0)$"
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionVideoSolver
|
||||||
|
MAX_STEPS: 2000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_cogvideox_2b_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 4
|
||||||
|
FPS: 8
|
||||||
|
SHARDING_STRATEGY: full_shard
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
SAVE_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionCogVideoX
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
PARAMETERIZATION: v
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA: 3.0
|
||||||
|
ZERO_TERMINAL_SNR: True
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
SCALING_FACTOR_IMAGE: 1.15258426
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 3.0
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@transformer/diffusion_pytorch_model.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 30
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 30
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: False
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: False
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@vae/diffusion_pytorch_model.safetensors
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SEED: 42
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
NUM_FRAMES: 49
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: Adam
|
||||||
|
LEARNING_RATE: 1e-3
|
||||||
|
BETAS: [ 0.9, 0.95 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 0.0
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
# LR_SCHEDULER:
|
||||||
|
# NAME: StepAnnealingLR
|
||||||
|
# WARMUP_STEPS: 200
|
||||||
|
# TOTAL_STEPS: 2000
|
||||||
|
# DECAY_MODE: 'cosine'
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: VideoGenDatasetOTF
|
||||||
|
MODE: train
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
FIELDS: [ 'video_path', 'width', 'height', 'prompt' ]
|
||||||
|
PATH_PREFIX:
|
||||||
|
DATA_FILE:
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'video', 'video_latent', "prompt" ]
|
||||||
|
META_KEYS: [ ]
|
||||||
|
MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-2b@vae/diffusion_pytorch_model.safetensors
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "A girl riding a bike.", "A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. The panda's fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, casting a gentle glow on the scene. The panda's face is expressive, showing concentration and joy as it plays. The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical atmosphere of this unique musical performance." ]
|
||||||
|
IMAGE_SIZE: [ 480, 720 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
# USE_NUM: 8
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
PRIORITY: 20
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 1000
|
||||||
|
PRIORITY: 40
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
SEED: 42
|
||||||
|
TENSOR_PARALLEL_SIZE: 1
|
||||||
|
PIPELINE_PARALLEL_SIZE: 1
|
||||||
|
SYS_ENVS:
|
||||||
|
TORCH_CUDNN_V8_API_ENABLED: '1'
|
||||||
|
TOKENIZERS_PARALLELISM: 'false'
|
||||||
|
TF_CPP_MIN_LOG_LEVEL: '3'
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF: 'expandable_segments:True'
|
||||||
|
META:
|
||||||
|
VERSION: 'COGVIDEOX_5B'
|
||||||
|
DESCRIPTION: "cogvideox 5b"
|
||||||
|
IS_DEFAULT: False
|
||||||
|
IS_SHARE: True
|
||||||
|
INFERENCE_PARAS:
|
||||||
|
INFERENCE_BATCH_SIZE: 1
|
||||||
|
INFERENCE_PREFIX: ""
|
||||||
|
DEFAULT_SAMPLER: "ddim"
|
||||||
|
DEFAULT_SAMPLE_STEPS: 50
|
||||||
|
INFERENCE_N_PROMPT: ""
|
||||||
|
RESOLUTION: [ 480, 720 ]
|
||||||
|
PARAS:
|
||||||
|
- TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [ 480, 720 ]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: False
|
||||||
|
TUNER: FULL
|
||||||
|
- TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [ 480, 720 ]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: True
|
||||||
|
TUNER: LORA
|
||||||
|
#
|
||||||
|
TUNERS:
|
||||||
|
LORA:
|
||||||
|
- NAME: SwiftLoRA
|
||||||
|
R: 64
|
||||||
|
LORA_ALPHA: 64
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.to_k|.to_q|.to_v|.to_out.0)$"
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionVideoSolver
|
||||||
|
MAX_STEPS: 2000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_cogvideox_5b_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 4
|
||||||
|
FPS: 8
|
||||||
|
SHARDING_STRATEGY: full_shard
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
SAVE_MODULES: [ 'model', 'cond_stage_model.model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionCogVideoX
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
PARAMETERIZATION: v
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA: 3.0
|
||||||
|
ZERO_TERMINAL_SNR: True
|
||||||
|
SCALE_FACTOR_SPATIAL: 8
|
||||||
|
SCALE_FACTOR_TEMPORAL: 4
|
||||||
|
SCALING_FACTOR_IMAGE: 0.7 # 5b diff
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: BaseDiffusion
|
||||||
|
PREDICTION_TYPE: v
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: ScaledLinearScheduler
|
||||||
|
BETA_MIN: 0.00085
|
||||||
|
BETA_MAX: 0.012
|
||||||
|
SNR_SHIFT_SCALE: 1.0 # 5b diff
|
||||||
|
RESCALE_BETAS_ZERO_SNR: True
|
||||||
|
DIFFUSION_SAMPLERS:
|
||||||
|
NAME: DDIMSampler
|
||||||
|
DISCRETIZATION_TYPE: trailing
|
||||||
|
ETA: 0.0
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: CogVideoXTransformer3DModel
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: # 5b diff
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b@transformer/diffusion_pytorch_model-00001-of-00002.safetensors
|
||||||
|
- ms://AI-ModelScope/CogVideoX-5b@transformer/diffusion_pytorch_model-00002-of-00002.safetensors
|
||||||
|
NUM_ATTENTION_HEADS: 48 # 5b diff
|
||||||
|
ATTENTION_HEAD_DIM: 64
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
FLIP_SIN_TO_COS: True
|
||||||
|
FREQ_SHIFT: 0
|
||||||
|
TIME_EMBED_DIM: 512
|
||||||
|
TEXT_EMBED_DIM: 4096
|
||||||
|
NUM_LAYERS: 42 # 5b diff
|
||||||
|
DROPOUT: 0.0
|
||||||
|
ATTENTION_BIAS: True
|
||||||
|
SAMPLE_WIDTH: 90
|
||||||
|
SAMPLE_HEIGHT: 60
|
||||||
|
SAMPLE_FRAMES: 49
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
MAX_TEXT_SEQ_LENGTH: 226
|
||||||
|
ACTIVATION_FN: "gelu-approximate"
|
||||||
|
TIMESTEP_ACTIVATION_FN: "silu"
|
||||||
|
NORM_ELEMENTWISE_AFFINE: True
|
||||||
|
NORM_EPS: 1e-5
|
||||||
|
SPATIAL_INTERPOLATION_SCALE: 1.875
|
||||||
|
TEMPORAL_INTERPOLATION_SCALE: 1.0
|
||||||
|
USE_ROTARY_POSITIONAL_EMBEDDINGS: True # 5b diff
|
||||||
|
USE_LEARNED_POSITIONAL_EMBEDDINGS: False
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-5b@vae/diffusion_pytorch_model.safetensors # 5b diff
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/t5-v1_1-xxl
|
||||||
|
LENGTH: 226
|
||||||
|
CLEAN:
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SEED: 42
|
||||||
|
GUIDE_SCALE: 6.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
NUM_FRAMES: 49
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: Adam
|
||||||
|
LEARNING_RATE: 1e-3
|
||||||
|
BETAS: [ 0.9, 0.95 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 0.0
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
# LR_SCHEDULER:
|
||||||
|
# NAME: StepAnnealingLR
|
||||||
|
# WARMUP_STEPS: 200
|
||||||
|
# TOTAL_STEPS: 2000
|
||||||
|
# DECAY_MODE: 'cosine'
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: VideoGenDatasetOTF
|
||||||
|
MODE: train
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
FIELDS: [ 'video_path', 'width', 'height', 'prompt' ]
|
||||||
|
PATH_PREFIX:
|
||||||
|
DATA_FILE:
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'video', 'video_latent', "prompt" ]
|
||||||
|
META_KEYS: [ ]
|
||||||
|
MODEL:
|
||||||
|
NAME: AutoencoderKLCogVideoX
|
||||||
|
DTYPE: bfloat16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/CogVideoX-5b@vae/diffusion_pytorch_model.safetensors
|
||||||
|
SAMPLE_HEIGHT: 480
|
||||||
|
SAMPLE_WIDTH: 720
|
||||||
|
USE_QUANT_CONV: False
|
||||||
|
USE_POST_QUANT_CONV: False
|
||||||
|
USE_SLICING: True
|
||||||
|
USE_TILING: True
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
ENCODER:
|
||||||
|
NAME: CogVideoXEncoder3D
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D", "CogVideoXDownBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
DECODER:
|
||||||
|
NAME: CogVideoXDecoder3D
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
OUT_CHANNELS: 3
|
||||||
|
UP_BLOCK_TYPES: [ "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D", "CogVideoXUpBlock3D" ]
|
||||||
|
BLOCK_OUT_CHANNELS: [ 128, 256, 256, 512 ]
|
||||||
|
LAYERS_PER_BLOCK: 3
|
||||||
|
ACT_FN: "silu"
|
||||||
|
NORM_EPS: 1e-6
|
||||||
|
NORM_NUM_GROUPS: 32
|
||||||
|
DROPOUT: 0.0
|
||||||
|
PAD_MODE: "first"
|
||||||
|
TEMPORAL_COMPRESSION_RATIO: 4
|
||||||
|
GRADIENT_CHECKPOINTING: True
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "A girl riding a bike.", "A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. The panda's fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, casting a gentle glow on the scene. The panda's face is expressive, showing concentration and joy as it plays. The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical atmosphere of this unique musical performance." ]
|
||||||
|
IMAGE_SIZE: [ 480, 720 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
# USE_NUM: 8
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
PRIORITY: 20
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 1000
|
||||||
|
PRIORITY: 40
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
META:
|
||||||
|
VERSION: 'FLUX1.0_DEV'
|
||||||
|
DESCRIPTION: "flux 1.0 dev"
|
||||||
|
IS_DEFAULT: True
|
||||||
|
IS_SHARE: True
|
||||||
|
INFERENCE_PARAS:
|
||||||
|
INFERENCE_BATCH_SIZE: 1
|
||||||
|
INFERENCE_PREFIX: ""
|
||||||
|
DEFAULT_SAMPLER: "flow_euler"
|
||||||
|
DEFAULT_SAMPLE_STEPS: 50
|
||||||
|
INFERENCE_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
PARAS:
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: False
|
||||||
|
TUNER: FULL
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: True
|
||||||
|
TUNER: LORA
|
||||||
|
#
|
||||||
|
TUNERS:
|
||||||
|
LORA:
|
||||||
|
-
|
||||||
|
NAME: SwiftLoRA
|
||||||
|
R: 4
|
||||||
|
LORA_ALPHA: 4
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "(model.double_blocks.*(.qkv|.proj|.img_mod.lin|.txt_mod.lin))|(model.single_blocks.*(.linear1|.linear2|.modulation.lin))$"
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
MAX_STEPS: 100000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_flux_dev_1024_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 16
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.t5_model' ] #
|
||||||
|
SAVE_MODULES: [ 'model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
|
||||||
|
FREEZE:
|
||||||
|
|
||||||
|
TUNER:
|
||||||
|
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionFlux
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
SAMPLER_SCHEDULER:
|
||||||
|
NAME: FlowMatchFluxShiftScheduler
|
||||||
|
SHIFT: False
|
||||||
|
SIGMOID_SCALE: 1
|
||||||
|
BASE_SHIFT: 0.5
|
||||||
|
MAX_SHIFT: 1.15
|
||||||
|
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: Flux
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@flux1-dev.safetensors
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
NUM_HEADS: 24
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
THETA: 10000
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
GUIDANCE_EMBED: False
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
QKV_BIAS: True
|
||||||
|
DEPTH: 19
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-dev@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
T5_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder_2/
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer_2/
|
||||||
|
MAX_LENGTH: 512
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
CLIP_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-dev@text_encoder/
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-dev@tokenizer/
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
|
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLE_STEPS: 50
|
||||||
|
SAMPLER: flow_euler
|
||||||
|
SEED: 2024
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
SHIFT: True
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 2
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
META:
|
||||||
|
VERSION: 'FLUX1.0_SCHNELL'
|
||||||
|
DESCRIPTION: "flux 1.0 schnell"
|
||||||
|
IS_DEFAULT: False
|
||||||
|
IS_SHARE: True
|
||||||
|
INFERENCE_PARAS:
|
||||||
|
INFERENCE_BATCH_SIZE: 1
|
||||||
|
INFERENCE_PREFIX: ""
|
||||||
|
DEFAULT_SAMPLER: "flow_euler"
|
||||||
|
DEFAULT_SAMPLE_STEPS: 4
|
||||||
|
INFERENCE_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
PARAS:
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: False
|
||||||
|
TUNER: FULL
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 1
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 89000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
IS_DEFAULT: True
|
||||||
|
TUNER: LORA
|
||||||
|
#
|
||||||
|
TUNERS:
|
||||||
|
LORA:
|
||||||
|
-
|
||||||
|
NAME: SwiftLoRA
|
||||||
|
R: 4
|
||||||
|
LORA_ALPHA: 4
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "(model.double_blocks.*(.qkv|.proj|.img_mod.lin|.txt_mod.lin))|(model.single_blocks.*(.linear1|.linear2|.modulation.lin))$"
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
MAX_STEPS: 100000
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: bfloat16
|
||||||
|
USE_FAIRSCALE: False
|
||||||
|
USE_FSDP: True
|
||||||
|
LOAD_MODEL_ONLY: False
|
||||||
|
RESUME_FROM:
|
||||||
|
WORK_DIR: ./cache/save_data/dit_flux_schnell_1024_lora
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
EVAL_INTERVAL: 100
|
||||||
|
LOG_TRAIN_NUM: 16
|
||||||
|
ENABLE_GRADSCALER: False
|
||||||
|
USE_SCALER: False
|
||||||
|
FSDP_REDUCE_DTYPE: float32
|
||||||
|
FSDP_BUFFER_DTYPE: float32
|
||||||
|
FSDP_SHARD_MODULES: [ 'model', 'cond_stage_model.t5_model' ]
|
||||||
|
SAVE_MODULES: [ 'model']
|
||||||
|
TRAIN_MODULES: ['model']
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
|
||||||
|
FREEZE:
|
||||||
|
|
||||||
|
TUNER:
|
||||||
|
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionFlux
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
USE_EMA: False
|
||||||
|
EVAL_EMA: False
|
||||||
|
DIFFUSION:
|
||||||
|
NAME: DiffusionFluxRF
|
||||||
|
PREDICTION_TYPE: raw
|
||||||
|
NOISE_SCHEDULER:
|
||||||
|
NAME: FlowMatchSigmaScheduler
|
||||||
|
WEIGHTING_SCHEME: logit_normal
|
||||||
|
SHIFT: 3.0
|
||||||
|
LOGIT_MEAN: 0.0
|
||||||
|
LOGIT_STD: 1.0
|
||||||
|
MODE_SCALE: 1.29
|
||||||
|
SAMPLER_SCHEDULER:
|
||||||
|
NAME: FlowMatchFluxShiftScheduler
|
||||||
|
SHIFT: False
|
||||||
|
SIGMOID_SCALE: 1
|
||||||
|
BASE_SHIFT: 0.5
|
||||||
|
MAX_SHIFT: 1.15
|
||||||
|
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: Flux
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-schnell@flux1-schnell.safetensors
|
||||||
|
IN_CHANNELS: 64
|
||||||
|
HIDDEN_SIZE: 3072
|
||||||
|
NUM_HEADS: 24
|
||||||
|
AXES_DIM: [ 16, 56, 56 ]
|
||||||
|
THETA: 10000
|
||||||
|
VEC_IN_DIM: 768
|
||||||
|
GUIDANCE_EMBED: False
|
||||||
|
CONTEXT_IN_DIM: 4096
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
QKV_BIAS: True
|
||||||
|
DEPTH: 19
|
||||||
|
DEPTH_SINGLE_BLOCKS: 38
|
||||||
|
USE_GRAD_CHECKPOINT: True
|
||||||
|
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKLFlux
|
||||||
|
EMBED_DIM: 16
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/FLUX.1-schnell@ae.safetensors
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 8
|
||||||
|
USE_CONV: False
|
||||||
|
SCALE_FACTOR: 0.3611
|
||||||
|
SHIFT_FACTOR: 0.1159
|
||||||
|
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5PlusClipFluxEmbedder
|
||||||
|
T5_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: T5EncoderModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-schnell@text_encoder_2/
|
||||||
|
HF_TOKENIZER_CLS: T5Tokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-schnell@tokenizer_2/
|
||||||
|
MAX_LENGTH: 256
|
||||||
|
OUTPUT_KEY: last_hidden_state
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: False
|
||||||
|
CLEAN: whitespace
|
||||||
|
CLIP_MODEL:
|
||||||
|
NAME: HFEmbedder
|
||||||
|
HF_MODEL_CLS: CLIPTextModel
|
||||||
|
MODEL_PATH: ms://AI-ModelScope/FLUX.1-schnell@text_encoder/
|
||||||
|
HF_TOKENIZER_CLS: CLIPTokenizer
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/FLUX.1-schnell@tokenizer/
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
OUTPUT_KEY: pooler_output
|
||||||
|
D_TYPE: bfloat16
|
||||||
|
BATCH_INFER: True
|
||||||
|
CLEAN: whitespace
|
||||||
|
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLE_STEPS: 4
|
||||||
|
SAMPLER: flow_euler
|
||||||
|
SEED: 2024
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
GUIDE_SCALE: 3.5
|
||||||
|
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 4e-4
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 2
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
PRIORITY: 0
|
||||||
|
- NAME: BackwardHook
|
||||||
|
PRIORITY: 10
|
||||||
|
- NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
- NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
EVAL_HOOKS:
|
||||||
|
- NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,264 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
META:
|
||||||
|
VERSION: 'PIXART_ALPHA'
|
||||||
|
DESCRIPTION: "PIXART ALPHA"
|
||||||
|
IS_DEFAULT: False
|
||||||
|
IS_SHARE: True
|
||||||
|
INFERENCE_PARAS:
|
||||||
|
INFERENCE_BATCH_SIZE: 1
|
||||||
|
INFERENCE_PREFIX: ""
|
||||||
|
DEFAULT_SAMPLER: "ddim"
|
||||||
|
DEFAULT_SAMPLE_STEPS: 20
|
||||||
|
INFERENCE_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
PARAS:
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 2
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 29000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 0.0001
|
||||||
|
IS_DEFAULT: False
|
||||||
|
TUNER: FULL
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 2
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 29000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 0.0001
|
||||||
|
IS_DEFAULT: True
|
||||||
|
TUNER: LORA
|
||||||
|
#
|
||||||
|
TUNERS:
|
||||||
|
LORA:
|
||||||
|
-
|
||||||
|
NAME: SwiftLoRA
|
||||||
|
R: 256
|
||||||
|
LORA_ALPHA: 256
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.q|.k|.v|.o|mlp.fc1|mlp.fc2)$"
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 1000
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: -1
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: cache/scepter_ui/self_train/dit/pixart_alpha_pro
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
FREEZE:
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionPixart
|
||||||
|
PARAMETERIZATION: eps
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 0.18215
|
||||||
|
SIZE_FACTOR: 8
|
||||||
|
DECODER_BIAS: 0.5
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "linear"
|
||||||
|
"BETA_MIN": 0.0001
|
||||||
|
"BETA_MAX": 0.02
|
||||||
|
USE_EMA: False
|
||||||
|
LOAD_REFINER: False
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: PixArt
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@PixArt-XL-2-1024-MS.pth
|
||||||
|
INPUT_SIZE: 128
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
IN_CHANNELS: 4
|
||||||
|
HIDDEN_SIZE: 1152
|
||||||
|
DEPTH: 28
|
||||||
|
NUM_HEADS: 16
|
||||||
|
MLP_RATIO: 4.0
|
||||||
|
CLASS_DROPOUT_PROB: 0.1
|
||||||
|
PRED_SIGMA: True
|
||||||
|
DROP_PATH: 0.0
|
||||||
|
WINDOW_DIZE: 0
|
||||||
|
USE_REL_POS: False
|
||||||
|
CAPTION_CHANNELS: 4096
|
||||||
|
LEWEI_SCALE: 2
|
||||||
|
MODEL_MAX_LENGTH: 120
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-2-base@512-base-ema.safetensors
|
||||||
|
EMBED_DIM: 4
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 4
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/PixArt-alpha@t5-v1_1-xxl/
|
||||||
|
LENGTH: 120
|
||||||
|
CLEAN: heavy
|
||||||
|
USE_GRAD: False
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: ddim
|
||||||
|
SAMPLE_STEPS: 20
|
||||||
|
SEED: 2024
|
||||||
|
GUIDE_SCALE: 4.5
|
||||||
|
GUIDE_RESCALE: 0.5
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
RUN_TRAIN_N: False
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 0.0001
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a boy wearing a jacket", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 0
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
SHOW_GPU_MEM: True
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
ENV:
|
||||||
|
BACKEND: nccl
|
||||||
|
META:
|
||||||
|
VERSION: 'SD3'
|
||||||
|
DESCRIPTION: "SD3 ALPHA"
|
||||||
|
IS_DEFAULT: False
|
||||||
|
IS_SHARE: True
|
||||||
|
INFERENCE_PARAS:
|
||||||
|
INFERENCE_BATCH_SIZE: 1
|
||||||
|
INFERENCE_PREFIX: ""
|
||||||
|
DEFAULT_SAMPLER: "euler"
|
||||||
|
DEFAULT_SAMPLE_STEPS: 28
|
||||||
|
INFERENCE_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
PARAS:
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 2
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 29000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 1e-5
|
||||||
|
IS_DEFAULT: False
|
||||||
|
TUNER: FULL
|
||||||
|
-
|
||||||
|
TRAIN_BATCH_SIZE: 2
|
||||||
|
TRAIN_PREFIX: ""
|
||||||
|
TRAIN_N_PROMPT: ""
|
||||||
|
RESOLUTION: [1024, 1024]
|
||||||
|
MEMORY: 29000
|
||||||
|
EPOCHS: 50
|
||||||
|
SAVE_INTERVAL: 25
|
||||||
|
EPSEC: 0.818
|
||||||
|
LEARNING_RATE: 5e-5
|
||||||
|
IS_DEFAULT: True
|
||||||
|
TUNER: LORA
|
||||||
|
#
|
||||||
|
TUNERS:
|
||||||
|
LORA:
|
||||||
|
-
|
||||||
|
NAME: SwiftLoRA
|
||||||
|
R: 128
|
||||||
|
LORA_ALPHA: 128
|
||||||
|
LORA_DROPOUT: 0.0
|
||||||
|
BIAS: "none"
|
||||||
|
TARGET_MODULES: "model.*(.attn.qkv|.attn.proj|mlp.fc1|mlp.fc2)$"
|
||||||
|
#
|
||||||
|
SOLVER:
|
||||||
|
NAME: LatentDiffusionSolver
|
||||||
|
RESUME_FROM:
|
||||||
|
LOAD_MODEL_ONLY: True
|
||||||
|
USE_FSDP: False
|
||||||
|
SHARDING_STRATEGY:
|
||||||
|
USE_AMP: True
|
||||||
|
DTYPE: float16
|
||||||
|
CHANNELS_LAST: True
|
||||||
|
MAX_STEPS: 1000
|
||||||
|
MAX_EPOCHS: -1
|
||||||
|
NUM_FOLDS: 1
|
||||||
|
ACCU_STEP: 1
|
||||||
|
EVAL_INTERVAL: -1
|
||||||
|
RESCALE_LR: False
|
||||||
|
#
|
||||||
|
WORK_DIR: cache/scepter_ui/self_train/dit/sd3
|
||||||
|
LOG_FILE: std_log.txt
|
||||||
|
#
|
||||||
|
FILE_SYSTEM:
|
||||||
|
- NAME: "ModelscopeFs"
|
||||||
|
TEMP_DIR: "./cache/cache_data"
|
||||||
|
#
|
||||||
|
FREEZE:
|
||||||
|
#
|
||||||
|
TUNER:
|
||||||
|
#
|
||||||
|
MODEL:
|
||||||
|
NAME: LatentDiffusionSD3
|
||||||
|
PARAMETERIZATION: rf
|
||||||
|
TIMESTEPS: 1000
|
||||||
|
MIN_SNR_GAMMA:
|
||||||
|
ZERO_TERMINAL_SNR: False
|
||||||
|
PRETRAINED_MODEL:
|
||||||
|
IGNORE_KEYS: [ ]
|
||||||
|
SCALE_FACTOR: 1.5305
|
||||||
|
SHIFT_FACTOR: 0.0609
|
||||||
|
DEFAULT_N_PROMPT:
|
||||||
|
SCHEDULE_ARGS:
|
||||||
|
"NAME": "shifted"
|
||||||
|
"SHIFT": 3
|
||||||
|
USE_EMA: False
|
||||||
|
T_WEIGHT: uniform
|
||||||
|
#
|
||||||
|
DIFFUSION_MODEL:
|
||||||
|
NAME: MMDiT
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
IGNORE_KEYS: '^first_stage_model.'
|
||||||
|
IN_CHANNELS: 16
|
||||||
|
PATCH_SIZE: 2
|
||||||
|
OUT_CHANNELS: 16
|
||||||
|
DEPTH: 24
|
||||||
|
INPUT_SIZE:
|
||||||
|
ADM_IN_CHANNELS: 2048
|
||||||
|
CONTEXT_EMBEDDER_CONFIG: { 'target': 'torch.nn.Linear', 'params': { 'in_features': 4096, 'out_features': 1536 } }
|
||||||
|
NUM_PATCHES: 36864
|
||||||
|
POS_EMBED_MAX_SIZE: 192
|
||||||
|
POS_EMBED_SCALING_FACTOR:
|
||||||
|
USE_CHECKPOINT: True
|
||||||
|
#
|
||||||
|
FIRST_STAGE_MODEL:
|
||||||
|
NAME: AutoencoderKL
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium@sd3_medium.safetensors
|
||||||
|
EMBED_DIM: 16
|
||||||
|
IGNORE_KEYS: '^model.diffusion_model.'
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
USE_CONV: False
|
||||||
|
#
|
||||||
|
ENCODER:
|
||||||
|
NAME: Encoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DOUBLE_Z: True
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
#
|
||||||
|
DECODER:
|
||||||
|
NAME: Decoder
|
||||||
|
CH: 128
|
||||||
|
OUT_CH: 3
|
||||||
|
NUM_RES_BLOCKS: 2
|
||||||
|
IN_CHANNELS: 3
|
||||||
|
ATTN_RESOLUTIONS: [ ]
|
||||||
|
CH_MULT: [ 1, 2, 4, 4 ]
|
||||||
|
Z_CHANNELS: 16
|
||||||
|
DROPOUT: 0.0
|
||||||
|
RESAMP_WITH_CONV: True
|
||||||
|
GIVE_PRE_END: False
|
||||||
|
TANH_OUT: False
|
||||||
|
#
|
||||||
|
COND_STAGE_MODEL:
|
||||||
|
NAME: SD3TextEmbedder
|
||||||
|
P_ZERO: 0.0
|
||||||
|
CLIP_L:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
CLIP_G:
|
||||||
|
NAME: FrozenCLIPEmbedder2
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_2
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_2
|
||||||
|
MAX_LENGTH: 77
|
||||||
|
FREEZE: True
|
||||||
|
LAYER: penultimate
|
||||||
|
RETURN_POOLED: True
|
||||||
|
USE_FINAL_LAYER_NORM: False
|
||||||
|
IS_TRAINABLE: False
|
||||||
|
T5_XXL:
|
||||||
|
NAME: T5EmbedderHF
|
||||||
|
PRETRAINED_MODEL: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@text_encoder_3
|
||||||
|
TOKENIZER_PATH: ms://AI-ModelScope/stable-diffusion-3-medium-diffusers@tokenizer_3
|
||||||
|
LENGTH: 256
|
||||||
|
CLEAN: whitespace
|
||||||
|
USE_GRAD: False
|
||||||
|
T5_DTYPE: float16
|
||||||
|
#
|
||||||
|
LOSS:
|
||||||
|
NAME: ReconstructLoss
|
||||||
|
LOSS_TYPE: l2
|
||||||
|
#
|
||||||
|
SAMPLE_ARGS:
|
||||||
|
SAMPLER: euler
|
||||||
|
SAMPLE_STEPS: 28
|
||||||
|
SEED: 1749023094
|
||||||
|
GUIDE_SCALE: 5.0
|
||||||
|
GUIDE_RESCALE: 0.0
|
||||||
|
DISCRETIZATION: trailing
|
||||||
|
RUN_TRAIN_N: False
|
||||||
|
#
|
||||||
|
OPTIMIZER:
|
||||||
|
NAME: AdamW
|
||||||
|
LEARNING_RATE: 5e-5
|
||||||
|
BETAS: [ 0.9, 0.999 ]
|
||||||
|
EPS: 1e-8
|
||||||
|
WEIGHT_DECAY: 1e-2
|
||||||
|
AMSGRAD: False
|
||||||
|
#
|
||||||
|
TRAIN_DATA:
|
||||||
|
NAME: ImageTextPairMSDataset
|
||||||
|
MODE: train
|
||||||
|
MS_DATASET_NAME: style_custom_dataset
|
||||||
|
MS_DATASET_NAMESPACE: damo
|
||||||
|
MS_DATASET_SUBNAME: 3D
|
||||||
|
PROMPT_PREFIX: ""
|
||||||
|
MS_DATASET_SPLIT: train
|
||||||
|
MS_REMAP_KEYS: { 'Image:FILE': 'Target:FILE' }
|
||||||
|
REPLACE_STYLE: False
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
SAMPLER:
|
||||||
|
NAME: LoopSampler
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: LoadImageFromFile
|
||||||
|
RGB_ORDER: RGB
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleResize
|
||||||
|
INTERPOLATION: bilinear
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: FlexibleCenterCrop
|
||||||
|
SIZE: [ 1024, 1024 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: ImageToTensor
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'img' ]
|
||||||
|
BACKEND: pillow
|
||||||
|
- NAME: Normalize
|
||||||
|
MEAN: [ 0.5, 0.5, 0.5 ]
|
||||||
|
STD: [ 0.5, 0.5, 0.5 ]
|
||||||
|
INPUT_KEY: [ 'img' ]
|
||||||
|
OUTPUT_KEY: [ 'image' ]
|
||||||
|
BACKEND: torchvision
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'image', 'prompt' ]
|
||||||
|
META_KEYS: [ 'data_key' ]
|
||||||
|
#
|
||||||
|
EVAL_DATA:
|
||||||
|
NAME: Text2ImageDataset
|
||||||
|
MODE: eval
|
||||||
|
PROMPT_FILE:
|
||||||
|
PROMPT_DATA: [ "a cat holds a blackboard that writes \"hello world\"", "a dog running on the lawn" ]
|
||||||
|
IMAGE_SIZE: [ 1024, 1024 ]
|
||||||
|
FIELDS: [ "prompt" ]
|
||||||
|
DELIMITER: '#;#'
|
||||||
|
PROMPT_PREFIX: ''
|
||||||
|
PIN_MEMORY: True
|
||||||
|
BATCH_SIZE: 1
|
||||||
|
NUM_WORKERS: 4
|
||||||
|
TRANSFORMS:
|
||||||
|
- NAME: Select
|
||||||
|
KEYS: [ 'index', 'prompt' ]
|
||||||
|
META_KEYS: [ 'image_size' ]
|
||||||
|
#
|
||||||
|
TRAIN_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: BackwardHook
|
||||||
|
PRIORITY: 1
|
||||||
|
-
|
||||||
|
NAME: LogHook
|
||||||
|
LOG_INTERVAL: 10
|
||||||
|
SHOW_GPU_MEM: True
|
||||||
|
-
|
||||||
|
NAME: TensorboardLogHook
|
||||||
|
-
|
||||||
|
NAME: CheckpointHook
|
||||||
|
INTERVAL: 10000
|
||||||
|
PRIORITY: 200
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
DISABLE_SNAPSHOT: True
|
||||||
|
#
|
||||||
|
EVAL_HOOKS:
|
||||||
|
-
|
||||||
|
NAME: ProbeDataHook
|
||||||
|
PROB_INTERVAL: 100
|
||||||
|
SAVE_LAST: True
|
||||||
|
SAVE_NAME_PREFIX: 'step'
|
||||||
|
SAVE_PROBE_PREFIX: 'image'
|
||||||
@@ -142,13 +142,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: -1
|
EVAL_INTERVAL: -1
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR:
|
WORK_DIR:
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
#
|
#
|
||||||
@@ -258,7 +259,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
@@ -267,7 +268,7 @@ SOLVER:
|
|||||||
TRAIN_DATA:
|
TRAIN_DATA:
|
||||||
NAME: ImageTextPairMSDataset
|
NAME: ImageTextPairMSDataset
|
||||||
MODE: train
|
MODE: train
|
||||||
MS_DATASET_NAME: cache/save_data/delogo/
|
MS_DATASET_NAME: ./cache/cache_data/delogo/
|
||||||
MS_DATASET_NAMESPACE: ""
|
MS_DATASET_NAMESPACE: ""
|
||||||
MS_DATASET_SPLIT: "train"
|
MS_DATASET_SPLIT: "train"
|
||||||
MS_DATASET_SUBNAME: ""
|
MS_DATASET_SUBNAME: ""
|
||||||
|
|||||||
@@ -147,6 +147,7 @@ SOLVER:
|
|||||||
MAX_EPOCHS: -1
|
MAX_EPOCHS: -1
|
||||||
# NUM_FOLDS DESCRIPTION: Num folds for training. TYPE: int default: 1
|
# NUM_FOLDS DESCRIPTION: Num folds for training. TYPE: int default: 1
|
||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
EVAL_INTERVAL: -1
|
EVAL_INTERVAL: -1
|
||||||
# WORK_DIR DESCRIPTION: Save dir of the training log or model. TYPE: str default: ''
|
# WORK_DIR DESCRIPTION: Save dir of the training log or model. TYPE: str default: ''
|
||||||
@@ -155,7 +156,7 @@ SOLVER:
|
|||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
TUNER:
|
TUNER:
|
||||||
# MODEL DESCRIPTION: TYPE: default: ''
|
# MODEL DESCRIPTION: TYPE: default: ''
|
||||||
MODEL:
|
MODEL:
|
||||||
@@ -538,7 +539,7 @@ SOLVER:
|
|||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
# NAME DESCRIPTION: TYPE: default: ''
|
# NAME DESCRIPTION: TYPE: default: ''
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.0064
|
LEARNING_RATE: 0.00001
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
AMSGRAD: False
|
AMSGRAD: False
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -13,8 +13,10 @@ TRAIN_PARAS:
|
|||||||
VALUES: [[256, 256], [320, 180], [180, 320],
|
VALUES: [[256, 256], [320, 180], [180, 320],
|
||||||
[512, 512], [640, 360], [360, 640],
|
[512, 512], [640, 360], [360, 640],
|
||||||
[768, 768], [960, 540], [540, 960],
|
[768, 768], [960, 540], [540, 960],
|
||||||
[1024, 1024], [1280, 720], [720, 1280]]
|
[1024, 1024], [1280, 720], [720, 1280],
|
||||||
|
[720, 480], [480, 720]]
|
||||||
DEFAULT: [1024, 1024]
|
DEFAULT: [1024, 1024]
|
||||||
EVAL_PROMPTS:
|
EVAL_PROMPTS:
|
||||||
- a boy wearing a jacket
|
- a boy wearing a jacket
|
||||||
- a dog running on the lawn
|
- a dog running on the lawn
|
||||||
|
SAVE_FILE_LOCAL_PATH: "cache/scepter_ui/datasets/train_data_from_list"
|
||||||
|
|||||||
@@ -137,13 +137,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: -1
|
EVAL_INTERVAL: -1
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR:
|
WORK_DIR:
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
#
|
#
|
||||||
@@ -249,7 +250,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -80,13 +80,14 @@ SOLVER:
|
|||||||
NUM_FOLDS: 1
|
NUM_FOLDS: 1
|
||||||
ACCU_STEP: 1
|
ACCU_STEP: 1
|
||||||
EVAL_INTERVAL: -1
|
EVAL_INTERVAL: -1
|
||||||
|
RESCALE_LR: False
|
||||||
#
|
#
|
||||||
WORK_DIR:
|
WORK_DIR:
|
||||||
LOG_FILE: std_log.txt
|
LOG_FILE: std_log.txt
|
||||||
#
|
#
|
||||||
FILE_SYSTEM:
|
FILE_SYSTEM:
|
||||||
NAME: "ModelscopeFs"
|
NAME: "ModelscopeFs"
|
||||||
TEMP_DIR: "./cache/data"
|
TEMP_DIR: "./cache/cache_data"
|
||||||
#
|
#
|
||||||
FREEZE:
|
FREEZE:
|
||||||
#
|
#
|
||||||
@@ -191,7 +192,7 @@ SOLVER:
|
|||||||
#
|
#
|
||||||
OPTIMIZER:
|
OPTIMIZER:
|
||||||
NAME: AdamW
|
NAME: AdamW
|
||||||
LEARNING_RATE: 0.064
|
LEARNING_RATE: 0.0001
|
||||||
BETAS: [ 0.9, 0.999 ]
|
BETAS: [ 0.9, 0.999 ]
|
||||||
EPS: 1e-8
|
EPS: 1e-8
|
||||||
WEIGHT_DECAY: 1e-2
|
WEIGHT_DECAY: 1e-2
|
||||||
|
|||||||
@@ -13,3 +13,5 @@ BASE_MODEL_VERSION:
|
|||||||
TUNER_TYPE: [ 'LORA', 'SCE', 'FULL' ]
|
TUNER_TYPE: [ 'LORA', 'SCE', 'FULL' ]
|
||||||
- BASE_MODEL: 'EDIT'
|
- BASE_MODEL: 'EDIT'
|
||||||
TUNER_TYPE: [ 'LORA', 'SCE', 'FULL' ]
|
TUNER_TYPE: [ 'LORA', 'SCE', 'FULL' ]
|
||||||
|
- BASE_MODEL: 'FLUX1.0_DEV'
|
||||||
|
TUNER_TYPE: [ 'LORA' ]
|
||||||
@@ -3,9 +3,21 @@
|
|||||||
from scepter.modules.annotator.base_annotator import GeneralAnnotator
|
from scepter.modules.annotator.base_annotator import GeneralAnnotator
|
||||||
from scepter.modules.annotator.canny import CannyAnnotator
|
from scepter.modules.annotator.canny import CannyAnnotator
|
||||||
from scepter.modules.annotator.color import ColorAnnotator
|
from scepter.modules.annotator.color import ColorAnnotator
|
||||||
|
from scepter.modules.annotator.degradation import DegradationAnnotator
|
||||||
|
from scepter.modules.annotator.doodle import DoodleAnnotator
|
||||||
|
from scepter.modules.annotator.gray import GrayAnnotator
|
||||||
from scepter.modules.annotator.hed import HedAnnotator
|
from scepter.modules.annotator.hed import HedAnnotator
|
||||||
from scepter.modules.annotator.identity import IdentityAnnotator
|
from scepter.modules.annotator.identity import IdentityAnnotator
|
||||||
|
from scepter.modules.annotator.informative_drawing import (
|
||||||
|
InfoDrawAnimeAnnotator, InfoDrawContourAnnotator,
|
||||||
|
InfoDrawOpenSketchAnnotator)
|
||||||
|
from scepter.modules.annotator.inpainting import InpaintingAnnotator
|
||||||
from scepter.modules.annotator.invert import InvertAnnotator
|
from scepter.modules.annotator.invert import InvertAnnotator
|
||||||
from scepter.modules.annotator.midas_op import MidasDetector
|
from scepter.modules.annotator.midas_op import MidasDetector
|
||||||
from scepter.modules.annotator.mlsd_op import MLSDdetector
|
from scepter.modules.annotator.mlsd_op import MLSDdetector
|
||||||
from scepter.modules.annotator.openpose import OpenposeAnnotator
|
from scepter.modules.annotator.openpose import OpenposeAnnotator
|
||||||
|
from scepter.modules.annotator.outpainting import OutpaintingAnnotator, OutpaintingResize
|
||||||
|
from scepter.modules.annotator.pidinet import PiDiAnnotator
|
||||||
|
from scepter.modules.annotator.segmentation import ESAMAnnotator
|
||||||
|
from scepter.modules.annotator.sketch import SketchAnnotator
|
||||||
|
from scepter.modules.annotator.lama import LamaAnnotator
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
import math
|
||||||
|
import random
|
||||||
|
from abc import ABCMeta
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
||||||
|
from scepter.modules.annotator.registry import ANNOTATORS
|
||||||
|
from scepter.modules.utils.config import Config, dict_to_yaml
|
||||||
|
|
||||||
|
|
||||||
|
def gaussian_noise_op(im, v):
|
||||||
|
from basicsr.data.degradations import random_add_gaussian_noise
|
||||||
|
noise_level = v.get('noise_level', [10, 20])
|
||||||
|
out = random_add_gaussian_noise(
|
||||||
|
im,
|
||||||
|
sigma_range=noise_level,
|
||||||
|
clip=True,
|
||||||
|
rounds=False,
|
||||||
|
gray_prob=0.4,
|
||||||
|
)
|
||||||
|
out = np.clip(out, 0.0, 1.0)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def resize_op(im, v):
|
||||||
|
scale = v.get('scale', [0.5, 0.8])
|
||||||
|
h, w = im.shape[:2]
|
||||||
|
scale = random.uniform(scale[0], scale[1])
|
||||||
|
h_, w_ = int(h * scale), int(w * scale)
|
||||||
|
mode = v.get('mode', 'nearest')
|
||||||
|
if mode == 'nearest':
|
||||||
|
interpolation = cv2.INTER_NEAREST
|
||||||
|
elif mode == 'bilinear':
|
||||||
|
interpolation = cv2.INTER_LINEAR
|
||||||
|
elif mode == 'bicubic':
|
||||||
|
interpolation = cv2.INTER_CUBIC
|
||||||
|
else:
|
||||||
|
interpolation = cv2.INTER_NEAREST
|
||||||
|
im = cv2.resize(im, (w_, h_), interpolation=interpolation)
|
||||||
|
out = cv2.resize(im, (w, h), interpolation=interpolation)
|
||||||
|
out = np.clip(out, 0.0, 1.0)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def jpeg_op(im, v):
|
||||||
|
from basicsr.data.degradations import add_jpg_compression
|
||||||
|
jpeg_level = v.get('jpeg_level', [50, 75])
|
||||||
|
v = int(random.uniform(jpeg_level[0], jpeg_level[1]))
|
||||||
|
out = add_jpg_compression(im, v)
|
||||||
|
out = np.clip(out, 0.0, 1.0)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def gaussian_blur_op(im, v):
|
||||||
|
from basicsr.data.degradations import random_mixed_kernels
|
||||||
|
kernel_range = v.get('kernel_size', [7, 9])
|
||||||
|
kernel_size = random.choice(kernel_range)
|
||||||
|
kernel_size = min(int(kernel_size) // 2 * 2 + 1, 21)
|
||||||
|
blur_sigma = v.get('sigma', [0.9, 1.0])
|
||||||
|
kernel = random_mixed_kernels(
|
||||||
|
('iso', 'aniso', 'generalized_iso', 'generalized_aniso', 'plateau_iso',
|
||||||
|
'plateau_aniso'), (0.45, 0.25, 0.12, 0.03, 0.12, 0.03),
|
||||||
|
kernel_size,
|
||||||
|
blur_sigma,
|
||||||
|
blur_sigma, [-math.pi, math.pi], [0.5, 2.0], [1, 1.5],
|
||||||
|
noise_range=None)
|
||||||
|
|
||||||
|
pad_size = (21 - kernel_size) // 2
|
||||||
|
kernel = np.pad(kernel, ((pad_size, pad_size), (pad_size, pad_size)))
|
||||||
|
out = cv2.filter2D(im, -1, kernel)
|
||||||
|
out = np.clip(out, 0.0, 1.0)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class DegradationAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
||||||
|
para_dict = {}
|
||||||
|
|
||||||
|
def __init__(self, cfg, logger=None):
|
||||||
|
super().__init__(cfg, logger=logger)
|
||||||
|
self.params = cfg.get('PARAMS', {
|
||||||
|
'gaussian_noise': {},
|
||||||
|
'resize': {},
|
||||||
|
'jpeg': {},
|
||||||
|
'gaussian_blur': {},
|
||||||
|
})
|
||||||
|
if not isinstance(self.params, dict):
|
||||||
|
self.params = Config.get_dict(self.params)
|
||||||
|
self.random_degradation = cfg.get('RANDOM_DEGRADATION', False)
|
||||||
|
|
||||||
|
def forward(self, image):
|
||||||
|
if isinstance(image, Image.Image):
|
||||||
|
image = np.array(image)
|
||||||
|
elif isinstance(image, torch.Tensor):
|
||||||
|
image = image.detach().cpu().numpy()
|
||||||
|
elif isinstance(image, np.ndarray):
|
||||||
|
image = image.copy()
|
||||||
|
else:
|
||||||
|
raise f'Unsurpport datatype{type(image)}, only surpport np.ndarray, torch.Tensor, Pillow Image.'
|
||||||
|
if np.max(image) > 1.0:
|
||||||
|
image = (image / 255.).astype(np.float32)
|
||||||
|
|
||||||
|
degradation_list = list(self.params.keys())
|
||||||
|
if self.random_degradation:
|
||||||
|
random.shuffle(degradation_list)
|
||||||
|
|
||||||
|
for degradation_type in degradation_list:
|
||||||
|
if degradation_type == 'gaussian_noise':
|
||||||
|
image = gaussian_noise_op(image, self.params[degradation_type])
|
||||||
|
elif degradation_type == 'resize':
|
||||||
|
image = resize_op(image, self.params[degradation_type])
|
||||||
|
elif degradation_type == 'jpeg':
|
||||||
|
image = jpeg_op(image, self.params[degradation_type])
|
||||||
|
elif degradation_type == 'gaussian_blur':
|
||||||
|
image = gaussian_blur_op(image, self.params[degradation_type])
|
||||||
|
else:
|
||||||
|
raise NotImplementedError(
|
||||||
|
f'ERROR: degradation_type: {degradation_type} is invalid.')
|
||||||
|
image = (image * 255.0).astype(np.uint8)
|
||||||
|
|
||||||
|
assert len(image.shape) < 4
|
||||||
|
return image
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_config_template():
|
||||||
|
return dict_to_yaml('ANNOTATORS',
|
||||||
|
__class__.__name__,
|
||||||
|
DegradationAnnotator.para_dict,
|
||||||
|
set_name=True)
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
from abc import ABCMeta
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
||||||
|
from scepter.modules.annotator.registry import ANNOTATORS
|
||||||
|
from scepter.modules.utils.config import dict_to_yaml
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class DoodleAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
||||||
|
para_dict = {}
|
||||||
|
|
||||||
|
def __init__(self, cfg, logger=None):
|
||||||
|
super().__init__(cfg, logger=logger)
|
||||||
|
self.processor_type = cfg.get('PROCESSOR_TYPE', 'pidinet_sketch')
|
||||||
|
processor_cfg = cfg.get('PROCESSOR_CFG', None)
|
||||||
|
if self.processor_type == 'pidinet_sketch':
|
||||||
|
self.pidinet_ins = ANNOTATORS.build(processor_cfg[0])
|
||||||
|
self.sketch_ins = ANNOTATORS.build(processor_cfg[1])
|
||||||
|
else:
|
||||||
|
raise 'Unsurpport PROCESSOR for DoodleAnnotator'
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
@torch.inference_mode()
|
||||||
|
@torch.autocast('cuda', enabled=False)
|
||||||
|
def forward(self, image):
|
||||||
|
if self.processor_type == 'pidinet_sketch':
|
||||||
|
pidinet_res = self.pidinet_ins(image)
|
||||||
|
sketch_res = self.sketch_ins(pidinet_res)
|
||||||
|
doodle_res = sketch_res
|
||||||
|
else:
|
||||||
|
raise 'Unsurpport PROCESSOR for DoodleAnnotator'
|
||||||
|
return doodle_res
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_config_template():
|
||||||
|
return dict_to_yaml('ANNOTATORS',
|
||||||
|
__class__.__name__,
|
||||||
|
DoodleAnnotator.para_dict,
|
||||||
|
set_name=True)
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
from abc import ABCMeta
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
from PIL import Image
|
||||||
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
||||||
|
from scepter.modules.annotator.registry import ANNOTATORS
|
||||||
|
from scepter.modules.utils.config import dict_to_yaml
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class GrayAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
||||||
|
para_dict = {}
|
||||||
|
|
||||||
|
def __init__(self, cfg, logger=None):
|
||||||
|
super().__init__(cfg, logger=logger)
|
||||||
|
|
||||||
|
def forward(self, image):
|
||||||
|
if isinstance(image, Image.Image):
|
||||||
|
image = np.array(image)
|
||||||
|
elif isinstance(image, torch.Tensor):
|
||||||
|
image = image.detach().cpu().numpy()
|
||||||
|
elif isinstance(image, np.ndarray):
|
||||||
|
image = image.copy()
|
||||||
|
else:
|
||||||
|
raise f'Unsurpport datatype{type(image)}, only surpport np.ndarray, torch.Tensor, Pillow Image.'
|
||||||
|
gray_map = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
||||||
|
return gray_map[..., None].repeat(3, axis=2)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_config_template():
|
||||||
|
return dict_to_yaml('ANNOTATORS',
|
||||||
|
__class__.__name__,
|
||||||
|
GrayAnnotator.para_dict,
|
||||||
|
set_name=True)
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
from abc import ABCMeta
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
import torch.nn as nn
|
||||||
|
from einops import rearrange
|
||||||
|
from torchvision.transforms import InterpolationMode
|
||||||
|
|
||||||
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
||||||
|
from scepter.modules.annotator.registry import ANNOTATORS
|
||||||
|
from scepter.modules.utils.config import dict_to_yaml
|
||||||
|
from scepter.modules.utils.distribute import we
|
||||||
|
from scepter.modules.utils.file_system import FS
|
||||||
|
|
||||||
|
norm_layer = nn.InstanceNorm2d
|
||||||
|
|
||||||
|
|
||||||
|
class ResidualBlock(nn.Module):
|
||||||
|
def __init__(self, in_features):
|
||||||
|
super(ResidualBlock, self).__init__()
|
||||||
|
|
||||||
|
conv_block = [
|
||||||
|
nn.ReflectionPad2d(1),
|
||||||
|
nn.Conv2d(in_features, in_features, 3),
|
||||||
|
norm_layer(in_features),
|
||||||
|
nn.ReLU(inplace=True),
|
||||||
|
nn.ReflectionPad2d(1),
|
||||||
|
nn.Conv2d(in_features, in_features, 3),
|
||||||
|
norm_layer(in_features)
|
||||||
|
]
|
||||||
|
|
||||||
|
self.conv_block = nn.Sequential(*conv_block)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
return x + self.conv_block(x)
|
||||||
|
|
||||||
|
|
||||||
|
class ContourInference(nn.Module):
|
||||||
|
def __init__(self, input_nc, output_nc, n_residual_blocks=9, sigmoid=True):
|
||||||
|
super(ContourInference, self).__init__()
|
||||||
|
|
||||||
|
# Initial convolution block
|
||||||
|
model0 = [
|
||||||
|
nn.ReflectionPad2d(3),
|
||||||
|
nn.Conv2d(input_nc, 64, 7),
|
||||||
|
norm_layer(64),
|
||||||
|
nn.ReLU(inplace=True)
|
||||||
|
]
|
||||||
|
self.model0 = nn.Sequential(*model0)
|
||||||
|
|
||||||
|
# Downsampling
|
||||||
|
model1 = []
|
||||||
|
in_features = 64
|
||||||
|
out_features = in_features * 2
|
||||||
|
for _ in range(2):
|
||||||
|
model1 += [
|
||||||
|
nn.Conv2d(in_features, out_features, 3, stride=2, padding=1),
|
||||||
|
norm_layer(out_features),
|
||||||
|
nn.ReLU(inplace=True)
|
||||||
|
]
|
||||||
|
in_features = out_features
|
||||||
|
out_features = in_features * 2
|
||||||
|
self.model1 = nn.Sequential(*model1)
|
||||||
|
|
||||||
|
model2 = []
|
||||||
|
# Residual blocks
|
||||||
|
for _ in range(n_residual_blocks):
|
||||||
|
model2 += [ResidualBlock(in_features)]
|
||||||
|
self.model2 = nn.Sequential(*model2)
|
||||||
|
|
||||||
|
# Upsampling
|
||||||
|
model3 = []
|
||||||
|
out_features = in_features // 2
|
||||||
|
for _ in range(2):
|
||||||
|
model3 += [
|
||||||
|
nn.ConvTranspose2d(in_features,
|
||||||
|
out_features,
|
||||||
|
3,
|
||||||
|
stride=2,
|
||||||
|
padding=1,
|
||||||
|
output_padding=1),
|
||||||
|
norm_layer(out_features),
|
||||||
|
nn.ReLU(inplace=True)
|
||||||
|
]
|
||||||
|
in_features = out_features
|
||||||
|
out_features = in_features // 2
|
||||||
|
self.model3 = nn.Sequential(*model3)
|
||||||
|
|
||||||
|
# Output layer
|
||||||
|
model4 = [nn.ReflectionPad2d(3), nn.Conv2d(64, output_nc, 7)]
|
||||||
|
if sigmoid:
|
||||||
|
model4 += [nn.Sigmoid()]
|
||||||
|
|
||||||
|
self.model4 = nn.Sequential(*model4)
|
||||||
|
|
||||||
|
def forward(self, x, cond=None):
|
||||||
|
out = self.model0(x)
|
||||||
|
out = self.model1(out)
|
||||||
|
out = self.model2(out)
|
||||||
|
out = self.model3(out)
|
||||||
|
out = self.model4(out)
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class InfoDrawContourAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
||||||
|
para_dict = {}
|
||||||
|
|
||||||
|
def __init__(self, cfg, logger=None):
|
||||||
|
super().__init__(cfg, logger=logger)
|
||||||
|
input_nc = cfg.get('INPUT_NC', 3)
|
||||||
|
output_nc = cfg.get('OUTPUT_NC', 1)
|
||||||
|
n_residual_blocks = cfg.get('N_RESIDUAL_BLOCKS', 3)
|
||||||
|
sigmoid = cfg.get('SIGMOID', True)
|
||||||
|
pretrained_model = cfg.get('PRETRAINED_MODEL', None)
|
||||||
|
|
||||||
|
self.model = ContourInference(input_nc, output_nc, n_residual_blocks,
|
||||||
|
sigmoid)
|
||||||
|
with FS.get_from(pretrained_model, wait_finish=True) as local_path:
|
||||||
|
self.model.load_state_dict(torch.load(local_path))
|
||||||
|
self.model = self.model.eval().requires_grad_(False).to(we.device_id)
|
||||||
|
|
||||||
|
@torch.no_grad()
|
||||||
|
@torch.inference_mode()
|
||||||
|
@torch.autocast('cuda', enabled=False)
|
||||||
|
def forward(self, image):
|
||||||
|
is_batch = False if len(image.shape) == 3 else True
|
||||||
|
if isinstance(image, torch.Tensor):
|
||||||
|
if len(image.shape) == 3:
|
||||||
|
image = rearrange(image, 'h w c -> 1 c h w')
|
||||||
|
B, C, H, W = image.shape
|
||||||
|
elif len(image.shape) == 4:
|
||||||
|
B, C, H, W = image.shape
|
||||||
|
else:
|
||||||
|
raise "Unsurpport input image's shape"
|
||||||
|
elif isinstance(image, np.ndarray):
|
||||||
|
image = torch.from_numpy(image.copy()).float()
|
||||||
|
if len(image.shape) == 3:
|
||||||
|
image = rearrange(image, 'h w c -> 1 c h w')
|
||||||
|
B, C, H, W = image.shape
|
||||||
|
elif len(image.shape) == 4:
|
||||||
|
B, C, H, W = image.shape
|
||||||
|
else:
|
||||||
|
raise "Unsurpport input image's shape"
|
||||||
|
else:
|
||||||
|
raise "Unsurpport input image's type"
|
||||||
|
|
||||||
|
image = image.float().div(255).to(we.device_id)
|
||||||
|
contour_map = self.model(image)
|
||||||
|
contour_map = (contour_map.squeeze(dim=1) * 255.0).clip(
|
||||||
|
0, 255).cpu().numpy().astype(np.uint8)
|
||||||
|
contour_map = contour_map[..., None].repeat(3, -1)
|
||||||
|
if not is_batch:
|
||||||
|
contour_map = contour_map.squeeze()
|
||||||
|
return contour_map
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_config_template():
|
||||||
|
return dict_to_yaml('ANNOTATORS',
|
||||||
|
__class__.__name__,
|
||||||
|
InfoDrawContourAnnotator.para_dict,
|
||||||
|
set_name=True)
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class InfoDrawAnimeAnnotator(InfoDrawContourAnnotator):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class InfoDrawOpenSketchAnnotator(InfoDrawContourAnnotator):
|
||||||
|
pass
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
import random
|
||||||
|
from abc import ABCMeta
|
||||||
|
from enum import Enum
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
||||||
|
from scepter.modules.annotator.registry import ANNOTATORS
|
||||||
|
from scepter.modules.utils.config import Config, dict_to_yaml
|
||||||
|
|
||||||
|
|
||||||
|
def invert_image(im):
|
||||||
|
im_arr = np.array(im)
|
||||||
|
mask_1 = im_arr == 0
|
||||||
|
mask_2 = im_arr == 255
|
||||||
|
im_arr[mask_1] = 255
|
||||||
|
im_arr[mask_2] = 0
|
||||||
|
new_im = im_arr
|
||||||
|
return new_im
|
||||||
|
|
||||||
|
|
||||||
|
class DrawMethod(Enum):
|
||||||
|
LINE = 'line'
|
||||||
|
CIRCLE = 'circle'
|
||||||
|
SQUARE = 'square'
|
||||||
|
|
||||||
|
|
||||||
|
def make_random_irregular_mask(shape,
|
||||||
|
max_angle=4,
|
||||||
|
max_len=60,
|
||||||
|
max_width=20,
|
||||||
|
min_times=0,
|
||||||
|
max_times=10,
|
||||||
|
draw_method=DrawMethod.LINE):
|
||||||
|
draw_method = DrawMethod(draw_method)
|
||||||
|
|
||||||
|
height, width = shape
|
||||||
|
mask = np.zeros((height, width), np.float32)
|
||||||
|
times = np.random.randint(min_times, max_times + 1)
|
||||||
|
for i in range(times):
|
||||||
|
start_x = np.random.randint(width)
|
||||||
|
start_y = np.random.randint(height)
|
||||||
|
for j in range(1 + np.random.randint(5)):
|
||||||
|
angle = 0.01 + np.random.randint(max_angle)
|
||||||
|
if i % 2 == 0:
|
||||||
|
angle = 2 * 3.1415926 - angle
|
||||||
|
length = 10 + np.random.randint(max_len)
|
||||||
|
brush_w = 5 + np.random.randint(max_width)
|
||||||
|
end_x = np.clip(
|
||||||
|
(start_x + length * np.sin(angle)).astype(np.int32), 0, width)
|
||||||
|
end_y = np.clip(
|
||||||
|
(start_y + length * np.cos(angle)).astype(np.int32), 0, height)
|
||||||
|
if draw_method == DrawMethod.LINE:
|
||||||
|
cv2.line(mask, (start_x, start_y), (end_x, end_y), 1.0,
|
||||||
|
brush_w)
|
||||||
|
elif draw_method == DrawMethod.CIRCLE:
|
||||||
|
cv2.circle(mask, (start_x, start_y),
|
||||||
|
radius=brush_w,
|
||||||
|
color=1.,
|
||||||
|
thickness=-1)
|
||||||
|
elif draw_method == DrawMethod.SQUARE:
|
||||||
|
radius = brush_w // 2
|
||||||
|
mask[start_y - radius:start_y + radius,
|
||||||
|
start_x - radius:start_x + radius] = 1
|
||||||
|
start_x, start_y = end_x, end_y
|
||||||
|
return mask[None, ...]
|
||||||
|
|
||||||
|
|
||||||
|
class RandomIrregularMaskGenerator:
|
||||||
|
def __init__(self,
|
||||||
|
max_angle=4,
|
||||||
|
max_len=60,
|
||||||
|
max_width=20,
|
||||||
|
min_times=0,
|
||||||
|
max_times=10,
|
||||||
|
ramp_kwargs=None,
|
||||||
|
draw_method=DrawMethod.LINE):
|
||||||
|
self.max_angle = max_angle
|
||||||
|
self.max_len = max_len
|
||||||
|
self.max_width = max_width
|
||||||
|
self.min_times = min_times
|
||||||
|
self.max_times = max_times
|
||||||
|
self.draw_method = draw_method
|
||||||
|
self.ramp = None
|
||||||
|
# self.ramp = LinearRamp(**ramp_kwargs) if ramp_kwargs is not None else None
|
||||||
|
|
||||||
|
def __call__(self, img, iter_i=None, raw_image=None):
|
||||||
|
coef = self.ramp(iter_i) if (self.ramp is not None) and (
|
||||||
|
iter_i is not None) else 1
|
||||||
|
cur_max_len = int(max(1, self.max_len * coef))
|
||||||
|
cur_max_width = int(max(1, self.max_width * coef))
|
||||||
|
cur_max_times = int(self.min_times + 1 +
|
||||||
|
(self.max_times - self.min_times) * coef)
|
||||||
|
return make_random_irregular_mask(img.shape[1:],
|
||||||
|
max_angle=self.max_angle,
|
||||||
|
max_len=cur_max_len,
|
||||||
|
max_width=cur_max_width,
|
||||||
|
min_times=self.min_times,
|
||||||
|
max_times=cur_max_times,
|
||||||
|
draw_method=self.draw_method)
|
||||||
|
|
||||||
|
|
||||||
|
def make_random_rectangle_mask(shape,
|
||||||
|
margin=10,
|
||||||
|
bbox_min_size=30,
|
||||||
|
bbox_max_size=100,
|
||||||
|
min_times=0,
|
||||||
|
max_times=3):
|
||||||
|
height, width = shape
|
||||||
|
mask = np.zeros((height, width), np.float32)
|
||||||
|
bbox_max_size = min(bbox_max_size, height - margin * 2, width - margin * 2)
|
||||||
|
times = np.random.randint(min_times, max_times + 1)
|
||||||
|
for i in range(times):
|
||||||
|
box_width = np.random.randint(bbox_min_size, bbox_max_size)
|
||||||
|
box_height = np.random.randint(bbox_min_size, bbox_max_size)
|
||||||
|
start_x = np.random.randint(margin, width - margin - box_width + 1)
|
||||||
|
start_y = np.random.randint(margin, height - margin - box_height + 1)
|
||||||
|
mask[start_y:start_y + box_height, start_x:start_x + box_width] = 1
|
||||||
|
return mask[None, ...]
|
||||||
|
|
||||||
|
|
||||||
|
class RandomRectangleMaskGenerator:
|
||||||
|
def __init__(self,
|
||||||
|
margin=10,
|
||||||
|
bbox_min_size=30,
|
||||||
|
bbox_max_size=100,
|
||||||
|
min_times=0,
|
||||||
|
max_times=3,
|
||||||
|
ramp_kwargs=None):
|
||||||
|
self.margin = margin
|
||||||
|
self.bbox_min_size = bbox_min_size
|
||||||
|
self.bbox_max_size = bbox_max_size
|
||||||
|
self.min_times = min_times
|
||||||
|
self.max_times = max_times
|
||||||
|
self.ramp = None
|
||||||
|
# self.ramp = LinearRamp(**ramp_kwargs) if ramp_kwargs is not None else None
|
||||||
|
|
||||||
|
def __call__(self, img, iter_i=None, raw_image=None):
|
||||||
|
coef = self.ramp(iter_i) if (self.ramp is not None) and (
|
||||||
|
iter_i is not None) else 1
|
||||||
|
cur_bbox_max_size = int(self.bbox_min_size + 1 +
|
||||||
|
(self.bbox_max_size - self.bbox_min_size) *
|
||||||
|
coef)
|
||||||
|
cur_max_times = int(self.min_times +
|
||||||
|
(self.max_times - self.min_times) * coef)
|
||||||
|
return make_random_rectangle_mask(img.shape[1:],
|
||||||
|
margin=self.margin,
|
||||||
|
bbox_min_size=self.bbox_min_size,
|
||||||
|
bbox_max_size=cur_bbox_max_size,
|
||||||
|
min_times=self.min_times,
|
||||||
|
max_times=cur_max_times)
|
||||||
|
|
||||||
|
|
||||||
|
class MixedMaskGenerator:
|
||||||
|
def __init__(self,
|
||||||
|
irregular_proba=1 / 3,
|
||||||
|
irregular_kwargs=None,
|
||||||
|
box_proba=1 / 3,
|
||||||
|
box_kwargs=None,
|
||||||
|
invert_proba=0):
|
||||||
|
self.probas = []
|
||||||
|
self.gens = []
|
||||||
|
|
||||||
|
if irregular_proba > 0:
|
||||||
|
self.probas.append(irregular_proba)
|
||||||
|
if irregular_kwargs is None:
|
||||||
|
irregular_kwargs = {}
|
||||||
|
else:
|
||||||
|
irregular_kwargs = dict(irregular_kwargs)
|
||||||
|
irregular_kwargs['draw_method'] = DrawMethod.LINE
|
||||||
|
self.gens.append(RandomIrregularMaskGenerator(**irregular_kwargs))
|
||||||
|
|
||||||
|
if box_proba > 0:
|
||||||
|
self.probas.append(box_proba)
|
||||||
|
if box_kwargs is None:
|
||||||
|
box_kwargs = {}
|
||||||
|
self.gens.append(RandomRectangleMaskGenerator(**box_kwargs))
|
||||||
|
|
||||||
|
self.probas = np.array(self.probas, dtype='float32')
|
||||||
|
self.probas /= self.probas.sum()
|
||||||
|
self.invert_proba = invert_proba
|
||||||
|
|
||||||
|
def __call__(self, img, iter_i=None, raw_image=None):
|
||||||
|
kind = np.random.choice(len(self.probas), p=self.probas)
|
||||||
|
gen = self.gens[kind]
|
||||||
|
result = gen(img, iter_i=iter_i, raw_image=raw_image)
|
||||||
|
if self.invert_proba > 0 and random.random() < self.invert_proba:
|
||||||
|
result = 1 - result
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class InpaintingAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
||||||
|
para_dict = {}
|
||||||
|
|
||||||
|
def __init__(self, cfg, logger=None):
|
||||||
|
super().__init__(cfg, logger=logger)
|
||||||
|
self.mask_cfg = cfg.get(
|
||||||
|
'MASK_CFG', {
|
||||||
|
'irregular_proba': 0.5,
|
||||||
|
'irregular_kwargs': {
|
||||||
|
'min_times': 4,
|
||||||
|
'max_times': 10,
|
||||||
|
'max_width': 100,
|
||||||
|
'max_angle': 4,
|
||||||
|
'max_len': 200
|
||||||
|
},
|
||||||
|
'box_proba': 0.5,
|
||||||
|
'box_kwargs': {
|
||||||
|
'margin': 0,
|
||||||
|
'bbox_min_size': 30,
|
||||||
|
'bbox_max_size': 150,
|
||||||
|
'max_times': 5,
|
||||||
|
'min_times': 1
|
||||||
|
}
|
||||||
|
})
|
||||||
|
self.mask_cfg = Config.get_dict(self.mask_cfg) if not isinstance(
|
||||||
|
self.mask_cfg, dict) else self.mask_cfg
|
||||||
|
self.mask_generator = MixedMaskGenerator(**self.mask_cfg)
|
||||||
|
self.return_mask = cfg.get('RETURN_MASK', False)
|
||||||
|
self.return_invert = cfg.get('RETURN_INVERT', True)
|
||||||
|
self.mask_color = cfg.get('MASK_COLOR', 0)
|
||||||
|
|
||||||
|
def forward(self,
|
||||||
|
image,
|
||||||
|
mask=None,
|
||||||
|
return_mask=None,
|
||||||
|
return_invert=None,
|
||||||
|
mask_color=None):
|
||||||
|
return_mask = return_mask if return_mask is not None else self.return_mask
|
||||||
|
return_invert = return_invert if return_invert is not None else self.return_invert
|
||||||
|
mask_color = mask_color if mask_color is not None else self.mask_color
|
||||||
|
if isinstance(image, Image.Image):
|
||||||
|
image = np.array(image)
|
||||||
|
elif isinstance(image, torch.Tensor):
|
||||||
|
image = image.detach().cpu().numpy()
|
||||||
|
elif isinstance(image, np.ndarray):
|
||||||
|
image = image.copy()
|
||||||
|
else:
|
||||||
|
raise f'Unsurpport datatype{type(image)}, only surpport np.ndarray, torch.Tensor, Pillow Image.'
|
||||||
|
|
||||||
|
if mask is not None:
|
||||||
|
if mask_color:
|
||||||
|
image[np.array(mask) == 255] = mask_color
|
||||||
|
else:
|
||||||
|
image[np.array(mask) == 255] = 0
|
||||||
|
else:
|
||||||
|
img = np.transpose(image, (2, 0, 1))
|
||||||
|
mask = self.mask_generator(img)
|
||||||
|
mask = (np.transpose(mask,
|
||||||
|
(1, 2, 0)).squeeze(-1) * 255).astype(np.uint8)
|
||||||
|
if return_invert:
|
||||||
|
mask = invert_image(mask)
|
||||||
|
colored_mask = np.zeros_like(image)
|
||||||
|
if mask_color: colored_mask[:] = mask_color
|
||||||
|
image = np.where(mask[:, :, np.newaxis] == 255, colored_mask,
|
||||||
|
image)
|
||||||
|
|
||||||
|
if return_mask:
|
||||||
|
ret_data = {'image': np.array(image), 'mask': np.array(mask)}
|
||||||
|
else:
|
||||||
|
ret_data = np.array(image)
|
||||||
|
return ret_data
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_config_template():
|
||||||
|
return dict_to_yaml('ANNOTATORS',
|
||||||
|
__class__.__name__,
|
||||||
|
InpaintingAnnotator.para_dict,
|
||||||
|
set_name=True)
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||||
|
from abc import ABCMeta
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import torch
|
||||||
|
from PIL import Image
|
||||||
|
from scepter.modules.annotator.base_annotator import BaseAnnotator
|
||||||
|
from scepter.modules.annotator.registry import ANNOTATORS
|
||||||
|
from scepter.modules.utils.config import dict_to_yaml
|
||||||
|
from scepter.modules.utils.distribute import we
|
||||||
|
from scepter.modules.utils.file_system import FS
|
||||||
|
|
||||||
|
|
||||||
|
def dilate_mask(mask, dilate_factor=15):
|
||||||
|
mask = mask.astype(np.uint8)
|
||||||
|
mask = cv2.dilate(mask,
|
||||||
|
np.ones((dilate_factor, dilate_factor), np.uint8),
|
||||||
|
iterations=1)
|
||||||
|
return mask
|
||||||
|
|
||||||
|
|
||||||
|
@ANNOTATORS.register_class()
|
||||||
|
class LamaAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
||||||
|
para_dict = {}
|
||||||
|
|
||||||
|
def __init__(self, cfg, logger=None):
|
||||||
|
super().__init__(cfg, logger=logger)
|
||||||
|
from modelscope.pipelines.builder import PIPELINES
|
||||||
|
from modelscope.pipelines.cv import ImageInpaintingPipeline
|
||||||
|
from modelscope.pipelines import pipeline
|
||||||
|
from modelscope.utils.constant import Tasks
|
||||||
|
from modelscope.metainfo import Pipelines
|
||||||
|
from modelscope.models.cv.image_inpainting.refinement import refine_predict
|
||||||
|
from torch.utils.data._utils.collate import default_collate
|
||||||
|
|
||||||
|
@PIPELINES.register_module(Tasks.image_inpainting,
|
||||||
|
module_name=Pipelines.image_inpainting +
|
||||||
|
'-v2')
|
||||||
|
class ImageInpaintingPipelineV2(ImageInpaintingPipeline):
|
||||||
|
def perform_inference(self, data):
|
||||||
|
px_budget = 9000000
|
||||||
|
batch = default_collate([data])
|
||||||
|
if self.refine:
|
||||||
|
assert 'unpad_to_size' in batch, 'Unpadded size is required for the refinement'
|
||||||
|
assert 'cuda' in str(
|
||||||
|
self.device), 'GPU is required for refinement'
|
||||||
|
gpu_ids = str(self.device).split(':')[-1]
|
||||||
|
cur_res = refine_predict(batch,
|
||||||
|
self.infer_model,
|
||||||
|
gpu_ids=gpu_ids,
|
||||||
|
modulo=self.pad_out_to_modulo,
|
||||||
|
n_iters=15,
|
||||||
|
lr=0.002,
|
||||||
|
min_side=512,
|
||||||
|
max_scales=3,
|
||||||
|
px_budget=px_budget)
|
||||||
|
cur_res = cur_res[0].permute(1, 2,
|
||||||
|
0).detach().cpu().numpy()
|
||||||
|
else:
|
||||||
|
with torch.no_grad():
|
||||||
|
batch = self.move_to_device(batch, self.device)
|
||||||
|
batch['mask'] = (batch['mask'] > 0) * 1
|
||||||
|
batch = self.infer_model(batch)
|
||||||
|
cur_res = batch['inpainted'][0].permute(
|
||||||
|
1, 2, 0).detach().cpu().numpy()
|
||||||
|
unpad_to_size = batch.get('unpad_to_size', None)
|
||||||
|
if unpad_to_size is not None:
|
||||||
|
orig_height, orig_width = unpad_to_size
|
||||||
|
cur_res = cur_res[:orig_height, :orig_width]
|
||||||
|
|
||||||
|
cur_res = np.clip(cur_res * 255, 0, 255).astype('uint8')
|
||||||
|
cur_res = cv2.cvtColor(cur_res, cv2.COLOR_RGB2BGR)
|
||||||
|
return cur_res
|
||||||
|
|
||||||
|
lama_model_dir = FS.get_dir_to_local_dir(cfg.PRETRAINED_MODEL)
|
||||||
|
self.lama_model = pipeline(Tasks.image_inpainting,
|
||||||
|
model=lama_model_dir,
|
||||||
|
pipeline_name=Pipelines.image_inpainting +
|
||||||
|
'-v2',
|
||||||
|
refine=True,
|
||||||
|
device='cuda:{}'.format(we.device_id))
|
||||||
|
|
||||||
|
def forward(self, image, mask):
|
||||||
|
mask = dilate_mask(mask, dilate_factor=19)
|
||||||
|
input_mask = Image.fromarray(mask)
|
||||||
|
mask_expanded = np.tile(np.expand_dims(mask, axis=-1), (1, 1, 3))
|
||||||
|
input_image_np = np.array(image)
|
||||||
|
input_image_np[mask_expanded == 255] = 0
|
||||||
|
input_image = Image.fromarray(input_image_np)
|
||||||
|
input = {
|
||||||
|
'img': input_image,
|
||||||
|
'mask': input_mask,
|
||||||
|
}
|
||||||
|
result = self.lama_model(input)
|
||||||
|
output_img = result['output_img']
|
||||||
|
return output_img[..., ::-1]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_config_template():
|
||||||
|
return dict_to_yaml('ANNOTATORS',
|
||||||
|
__class__.__name__,
|
||||||
|
LamaAnnotator.para_dict,
|
||||||
|
set_name=True)
|
||||||
@@ -9,9 +9,10 @@ import os
|
|||||||
from abc import ABCMeta
|
from abc import ABCMeta
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
import cv2
|
import cv2
|
||||||
import matplotlib
|
import matplotlib
|
||||||
import numpy as np
|
|
||||||
import torch
|
import torch
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
@@ -478,7 +479,7 @@ class Hand(object):
|
|||||||
map_ori = heatmap_avg[:, :, part]
|
map_ori = heatmap_avg[:, :, part]
|
||||||
one_heatmap = gaussian_filter(map_ori, sigma=3)
|
one_heatmap = gaussian_filter(map_ori, sigma=3)
|
||||||
binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8)
|
binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8)
|
||||||
# 全部小于阈值
|
|
||||||
if np.sum(binary) == 0:
|
if np.sum(binary) == 0:
|
||||||
all_peaks.append([0, 0])
|
all_peaks.append([0, 0])
|
||||||
continue
|
continue
|
||||||
@@ -788,7 +789,7 @@ class OpenposeAnnotator(BaseAnnotator, metaclass=ABCMeta):
|
|||||||
raise f'Unsurpport datatype{type(image)}, only surpport np.ndarray, torch.Tensor, Pillow Image.'
|
raise f'Unsurpport datatype{type(image)}, only surpport np.ndarray, torch.Tensor, Pillow Image.'
|
||||||
image = image[:, :, ::-1]
|
image = image[:, :, ::-1]
|
||||||
candidate, subset = self.body_estimation(image)
|
candidate, subset = self.body_estimation(image)
|
||||||
canvas = np.zeros_like(image)
|
canvas = np.zeros_like(image, order='C') # to check
|
||||||
canvas = draw_bodypose(canvas, candidate, subset)
|
canvas = draw_bodypose(canvas, candidate, subset)
|
||||||
if self.use_hand:
|
if self.use_hand:
|
||||||
hands_list = handDetect(candidate, subset, image)
|
hands_list = handDetect(candidate, subset, image)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user