From 945c5bbe23d9472e374467dd6cef857581ef1b4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jukka=20Sepp=C3=A4nen?=
 <40791699+kijai@users.noreply.github.com>
Date: Fri, 11 Oct 2024 01:32:21 +0300
Subject: [PATCH] update example

---
 ...n => pyramidflow_text2vid_example_01.json} | 147 ++++++------------
 .../pyramid_dit_for_video_gen_pipeline.py     |   2 +-
 2 files changed, 46 insertions(+), 103 deletions(-)
 rename examples/{text2vid_example_20GB.json => pyramidflow_text2vid_example_01.json} (71%)

diff --git a/examples/text2vid_example_20GB.json b/examples/pyramidflow_text2vid_example_01.json
similarity index 71%
rename from examples/text2vid_example_20GB.json
rename to examples/pyramidflow_text2vid_example_01.json
index 6798d8b..5f41ee5 100644
--- a/examples/text2vid_example_20GB.json
+++ b/examples/pyramidflow_text2vid_example_01.json
@@ -1,13 +1,13 @@
 {
-  "last_node_id": 23,
-  "last_link_id": 31,
+  "last_node_id": 27,
+  "last_link_id": 38,
   "nodes": [
     {
       "id": 8,
       "type": "PyramidFlowVAEDecode",
       "pos": {
-        "0": 1555,
-        "1": 510
+        "0": 1161,
+        "1": 873
       },
       "size": {
         "0": 315,
@@ -33,7 +33,7 @@
           "name": "images",
           "type": "IMAGE",
           "links": [
-            26
+            38
           ],
           "slot_index": 0
         }
@@ -46,67 +46,16 @@
         2
       ]
     },
-    {
-      "id": 20,
-      "type": "GetImageSizeAndCount",
-      "pos": {
-        "0": 1564,
-        "1": 666
-      },
-      "size": [
-        257.25838733667274,
-        86
-      ],
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "image",
-          "type": "IMAGE",
-          "link": 26
-        }
-      ],
-      "outputs": [
-        {
-          "name": "image",
-          "type": "IMAGE",
-          "links": [
-            27
-          ],
-          "slot_index": 0
-        },
-        {
-          "name": "1280 width",
-          "type": "INT",
-          "links": null
-        },
-        {
-          "name": "768 height",
-          "type": "INT",
-          "links": null
-        },
-        {
-          "name": "57 count",
-          "type": "INT",
-          "links": null
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "GetImageSizeAndCount"
-      },
-      "widgets_values": []
-    },
     {
       "id": 22,
       "type": "PyramidFlowTextEncode",
       "pos": {
-        "0": 551,
-        "1": 731
+        "0": 567,
+        "1": 757
       },
       "size": {
-        "0": 403.6938171386719,
-        "1": 219.07676696777344
+        "0": 434.50982666015625,
+        "1": 227.74803161621094
       },
       "flags": {},
       "order": 1,
@@ -131,7 +80,7 @@
         "Node name for S&R": "PyramidFlowTextEncode"
       },
       "widgets_values": [
-        "Drone view of waves crashing against the rugged cliffs along Big Sur’s garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore, hyper quality, Ultra HD, 8K",
+        "A campfire burning with flames and embers, gradually increasing in size and intensity before dying down towards the end, hyper quality, Ultra HD, 8K",
         "",
         false
       ]
@@ -140,8 +89,8 @@
       "id": 9,
       "type": "PyramidFlowSampler",
       "pos": {
-        "0": 1077,
-        "1": 509
+        "0": 1059,
+        "1": 497
       },
       "size": {
         "0": 411.5168151855469,
@@ -191,12 +140,12 @@
       "widgets_values": [
         1280,
         768,
-        20,
-        10,
-        8,
+        "20, 20, 20",
+        "10, 10, 10",
+        16,
         7,
-        4,
-        44664248661391,
+        5,
+        44664248661394,
         "fixed",
         ""
       ]
@@ -205,12 +154,12 @@
       "id": 5,
       "type": "DownloadAndLoadPyramidFlowModel",
       "pos": {
-        "0": 564,
-        "1": 509
+        "0": 576,
+        "1": 496
       },
       "size": {
-        "0": 361.3862609863281,
-        "1": 154
+        "0": 385.7839050292969,
+        "1": 202
       },
       "flags": {},
       "order": 0,
@@ -234,29 +183,31 @@
         "rain1011/pyramid-flow-sd3",
         "diffusion_transformer_768p",
         "bf16",
-        "fp16",
-        "bf16"
+        "bf16",
+        "bf16",
+        false,
+        false
       ]
     },
     {
       "id": 14,
       "type": "VHS_VideoCombine",
       "pos": {
-        "0": 1917,
-        "1": 479
+        "0": 1534,
+        "1": 490
       },
       "size": [
-        1698.6201716509577,
-        1331.1721029905746
+        1698.6201171875,
+        1331.1720703125
       ],
       "flags": {},
-      "order": 5,
+      "order": 4,
       "mode": 0,
       "inputs": [
         {
           "name": "images",
           "type": "IMAGE",
-          "link": 27
+          "link": 38
         },
         {
           "name": "audio",
@@ -296,14 +247,14 @@
         "crf": 19,
         "save_metadata": true,
         "pingpong": false,
-        "save_output": false,
+        "save_output": true,
         "videopreview": {
           "hidden": false,
           "paused": false,
           "params": {
-            "filename": "PyramidFlow_00001.mp4",
+            "filename": "PyramidFlow_00022.mp4",
             "subfolder": "",
-            "type": "temp",
+            "type": "output",
             "format": "video/h264-mp4",
             "frame_rate": 16
           },
@@ -337,22 +288,6 @@
       1,
       "LATENT"
     ],
-    [
-      26,
-      8,
-      0,
-      20,
-      0,
-      "IMAGE"
-    ],
-    [
-      27,
-      20,
-      0,
-      14,
-      0,
-      "IMAGE"
-    ],
     [
       30,
       5,
@@ -368,16 +303,24 @@
       9,
       1,
       "PYRAMIDFLOWPROMPT"
+    ],
+    [
+      38,
+      8,
+      0,
+      14,
+      0,
+      "IMAGE"
     ]
   ],
   "groups": [],
   "config": {},
   "extra": {
     "ds": {
-      "scale": 0.5644739300537773,
+      "scale": 0.6209213230591555,
       "offset": [
-        -47.76827969285131,
-        -150.5722569771329
+        -310.35152437401774,
+        -279.23206017956113
       ]
     }
   },
diff --git a/pyramid_dit/pyramid_dit_for_video_gen_pipeline.py b/pyramid_dit/pyramid_dit_for_video_gen_pipeline.py
index 78fb88d..704ef2e 100644
--- a/pyramid_dit/pyramid_dit_for_video_gen_pipeline.py
+++ b/pyramid_dit/pyramid_dit_for_video_gen_pipeline.py
@@ -75,7 +75,7 @@ class PyramidDiTForVideoGeneration:
                     if name != "pos_embedding":
                         param.data = param.data.to(model_dtype)
 
-        if fp8_fastmode:
+        if model_dtype in [torch.float8_e4m3fn, torch.float8_e4m3fn] and fp8_fastmode:
             from ..fp8_optimization import convert_fp8_linear
             convert_fp8_linear(self.dit, torch.bfloat16)