Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Fill
|
c993ce37a8 | ||
|
Fill
|
676d246968 | ||
|
Fill
|
6a13e65942 | ||
|
Fill
|
5182e7d9dd | ||
|
Fill
|
86252b7710 | ||
|
Fill
|
cb0ab72c7d | ||
|
Fill
|
7a52d86154 | ||
|
Fill
|
c75c65a998 | ||
|
Fill
|
a4e12cbf58 | ||
|
Fill
|
94d9766fdc | ||
|
Fill
|
8c8c83c0e4 | ||
|
Fill
|
45251efeba | ||
|
Fill
|
2ac242657c | ||
|
Fill
|
affd56f233 | ||
|
Fill
|
f454a74a69 | ||
|
Fill
|
0bcd5c3c7c | ||
|
Fill
|
a5e677107a | ||
|
Fill
|
eaf132bad9 | ||
|
Fill
|
4fa0c57808 | ||
|
Fill
|
2083ca698c | ||
|
Fill
|
a8d0b03542 | ||
|
Fill
|
4188c291a3 | ||
|
Fill
|
d803ebc8c8 | ||
|
Fill
|
87398a98fe | ||
|
Fill
|
ed38e4898a | ||
|
Fill
|
390a9f8dea | ||
|
Fill
|
15b7790999 | ||
|
Fill
|
6f5cfb75ef | ||
|
Fill
|
b7f0f3a34c | ||
|
Fill
|
d76f467fa6 | ||
|
Fill
|
fc6c3f99d4 | ||
|
Fill
|
599cd808ca | ||
|
Fill
|
3b97406b0c | ||
|
Fill
|
70b5a7e328 | ||
|
Fill
|
15115e23d0 | ||
|
Fill
|
0699c5c412 | ||
|
Fill
|
d37c1cecf6 | ||
|
Fill
|
3c00aa0f21 | ||
|
Fill
|
5e59c8c484 | ||
|
Fill
|
83d10e8b57 | ||
|
Fill
|
42ceb62a4c | ||
|
Fill
|
94397d7935 | ||
|
Fill
|
7262de69eb |
@@ -1,9 +0,0 @@
|
||||
* text=auto eol=lf
|
||||
*.sh text eol=lf
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.ttf binary
|
||||
*.otf binary
|
||||
*.zip binary
|
||||
@@ -1,21 +0,0 @@
|
||||
name: Publish to Comfy registry
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "pyproject.toml"
|
||||
|
||||
jobs:
|
||||
publish-node:
|
||||
name: Publish Custom Node to registry
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v4
|
||||
- name: Publish Custom Node
|
||||
uses: Comfy-Org/publish-node-action@main
|
||||
with:
|
||||
## Add your own personal access token to your Github Repository secrets and reference it here.
|
||||
personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}
|
||||
-185
@@ -1,185 +0,0 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.code-workspace
|
||||
.vscode/
|
||||
config.json
|
||||
ignore.txt
|
||||
.env
|
||||
__pycache__
|
||||
|
||||
# Cache directory
|
||||
cache/
|
||||
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
#*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
||||
.pdm.toml
|
||||
.pdm-python
|
||||
.pdm-build/
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
.idea/
|
||||
|
||||
# Claude Code folder
|
||||
.claude/
|
||||
|
||||
# Build artifacts
|
||||
node.zip
|
||||
*.zip
|
||||
temp_downloads/
|
||||
@@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for describing the origin of the Work and
|
||||
reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Support. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or support.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2024 Filliptm (Machine Delusions)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied. See the License for the specific language governing
|
||||
permissions and limitations under the License.
|
||||
@@ -1,618 +1,122 @@
|
||||
# 🏵️ Fill-Nodes
|
||||
|
||||
If you enjoy this project, consider supporting me on Patreon!
|
||||
<p align="left">
|
||||
<a href="https://www.patreon.com/c/Machinedelusions">
|
||||
<img src="images\\nodes\\Patreon.png" width="150px" alt="Patreon">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
Fill-Nodes is a versatile collection of custom nodes for ComfyUI that extends functionality across multiple domains. Features include advanced image processing, visual effects generation, comprehensive file handling (PDF creation/extraction, Google Drive integration), AI model interfaces (GPT, DALL-E, Hugging Face, Runway, Gemini, Fal.ai, and more), utility nodes for workflow enhancement, audio-reactive visual effects, and specialized tools for video processing, captioning, and batch operations. The pack provides both practical workflow solutions and creative tools within a unified node collection.
|
||||
|
||||
## 🔍 Table of Contents
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td valign="top">
|
||||
<ul>
|
||||
<li><a href="#image-nodes">🖼️ Image Nodes</a></li>
|
||||
<li><a href="#captioning-nodes">📝 Captioning Nodes</a></li>
|
||||
<li><a href="#vfx-nodes">✨ VFX Nodes</a></li>
|
||||
<li><a href="#utility-nodes">🛠️ Utility Nodes</a></li>
|
||||
<li><a href="#ksamplers-nodes">🎲 KSamplers Nodes</a></li>
|
||||
<li><a href="#pdf-nodes">📄 PDF Nodes</a></li>
|
||||
<li><a href="#gpt-nodes">🤖 GPT Nodes</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
<td valign="top">
|
||||
<ul>
|
||||
<li><a href="#ai-nodes">🧠 AI Nodes</a></li>
|
||||
<li><a href="#audio-nodes">🔊 Audio Nodes</a></li>
|
||||
<li><a href="#experimental-nodes">🧪 Experimental Nodes</a></li>
|
||||
<li><a href="#prompting-nodes">💬 Prompting Nodes</a></li>
|
||||
<li><a href="#file-operation-nodes">📂 File Operation Nodes</a></li>
|
||||
<li><a href="#google-drive-nodes">☁️ Google Drive Nodes</a></li>
|
||||
<li><a href="#api-tool-nodes">🔌 API Tool Nodes</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
<td valign="top">
|
||||
<ul>
|
||||
<li><a href="#hugging-face-nodes">🤗 Hugging Face Nodes</a></li>
|
||||
<li><a href="#loader-nodes">⏬ Loader Nodes</a></li>
|
||||
<li><a href="#discord-nodes">💬 Discord Nodes</a></li>
|
||||
<li><a href="#work-in-progress-nodes">🚧 Work-in-Progress Nodes</a></li>
|
||||
<li><a href="#game-nodes">🎮 Game Nodes</a></li>
|
||||
<li><a href="#video-nodes">🎬 Video Nodes</a></li>
|
||||
<li><a href="#kartel-nodes">📋 Kartel Nodes</a></li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-image-nodes">🖼️ Image Nodes</h2></summary>
|
||||
|
||||
> *Nodes for manipulating, analyzing, and working with images.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_AnimeLineExtractor` | Extracts line art from images using OpenCV's adaptive thresholding and morphological operations. Provides controls for line threshold, line width, detail level, noise reduction via Gaussian blur, and optional output inversion for white backgrounds. |
|
||||
| `FL_ApplyMask` | Applies a mask to an image's alpha channel by adding or modifying the alpha channel based on the provided mask. Automatically handles dimension mismatches between image and mask through interpolation and supports batch processing. |
|
||||
| `FL_BlackFrameReject` | Filters out images from a batch based on black pixel percentage using configurable threshold and black level detection. Supports multiple detection methods (luminance, average, RGB, all channels) and optional inversion logic, returning filtered images with count statistics. |
|
||||
| `FL_CoTracker` | CoTracker-based point/region tracking for video frames with forward/backward track integration, visibility-aware mask synthesis, configurable track radius, and empty-region detection for masked downstream sampling. |
|
||||
| `FL_CreateShapeImageOnPath` | Renders a batch of frames containing a configurable shape (circle / square / triangle) at JSON-defined per-frame positions, with size, color, opacity, blur and motion-blur controls. Produces both the IMAGE and a MASK. |
|
||||
| `FL_ImageAddNoise` | Adds random Gaussian noise to images with controllable strength (0.0-1.0) and seed for reproducible results. Uses torch.randn to generate noise and clips output to valid 0.0-1.0 range. |
|
||||
| `FL_ImageAddToBatch` | Combines two image batches into a single batch by concatenating them. Automatically resizes the second batch to match the first batch's dimensions using bicubic or area interpolation based on scaling direction. |
|
||||
| `FL_ImageAdjuster` | Adjusts image properties including hue rotation (-180 to 180), saturation, brightness, contrast (all -100 to 100), and sharpness (0 to 100) using PIL's ImageEnhance. Converts to HSV for hue/saturation adjustments and applies UnsharpMask filter for sharpness, with real-time preview sent to the frontend. |
|
||||
| `FL_ImageAspectCropper` | Crops images to a target aspect ratio (specified as width:height) by center-cropping either width or height as needed. Ensures resulting dimensions are even numbers for compatibility with video codecs. |
|
||||
| `FL_ImageBatch` | Dynamically combines multiple images into a single batch with configurable number of inputs (2-100). Uses ComfyUI's ImageBatch node internally and automatically resizes images to match the first image's dimensions. |
|
||||
| `FL_ImageBatchToImageList` | Splits a batched image tensor into a Python list of single-image tensors (one per batch index), the inverse of `FL_ImageListToImageBatch`. |
|
||||
| `FL_ImageBatchToGrid` | Arranges a batch of images into a grid layout with a specified number of images per row. Creates a single composite image with black background filling any empty grid positions. |
|
||||
| `FL_ImageBlank` | Creates a blank image with specified width, height, and RGB color values. Generates a solid color image as a tensor suitable for use in ComfyUI workflows. |
|
||||
| `FL_ImageCrop` | Crops images from specified directions (top, bottom, left, right, or combinations) by a specified pixel amount. Automatically validates crop amounts to prevent exceeding image dimensions. |
|
||||
| `FL_ImageDimensionDisplay` | Extracts and returns the width and height dimensions of an input image as a formatted string. Supports both tensor and PIL image formats. |
|
||||
| `FL_ImageListToImageBatch` | Converts a list of individual images into a single batched image tensor. Automatically resizes images to match the first image's dimensions using Lanczos interpolation if needed. |
|
||||
| `FL_ImageNotes` | Adds a text bar with custom text to images, positioned at top or bottom. Supports configurable bar height, text size, borders, and optional word wrapping with automatic height adjustment to fit wrapped text. |
|
||||
| `FL_ImageOverlay` | Composites an overlay image onto a base using a mask, with 10 alignment presets or custom X/Y offsets, scale presets, 5 blend modes (normal/multiply/screen/overlay/add), opacity, mask invert/feather, and clip-vs-extend-canvas boundary handling. Includes on-node preview toggle. |
|
||||
| `FL_ImagePicker` | Interactive batch image selector that pauses workflow execution and pops a modal UI so the user can pick which images to keep before the rest of the workflow runs. Configurable timeout. |
|
||||
| `FL_ImageRandomizer` | Randomly selects an image or video from a specified directory based on a seed value (using modulo operation for deterministic selection). Supports subdirectory searching and returns the selected media as a batch along with the file path and filename. |
|
||||
| `FL_ImageReplace` | Replaces a single image in a batch with a replacement image at the given index, with optional MASK input/output (replacement_mask + per-batch masks) and on-node preview. Handles channel-mismatch, negative-index, and clamping automatically. |
|
||||
| `FL_ImageToMask` | Extracts a MASK from an IMAGE by selecting a channel: `luminance` (Rec. 601 weights), `red`, `green`, `blue`, or `alpha` (with fallback to a ones-mask if no alpha channel exists). |
|
||||
| `FL_RegionPNGOverlay` | Draws one or more source-pixel regions on an image and randomly places PNG overlays from per-region folders. The widget shows and edits each region's X/Y position plus width and height for repeatable text/logo box placement, with optional vignette preview controls. |
|
||||
| `FL_ImageSelector` | Selects specific images from a batch using comma-separated indices or the 'last' keyword. Returns a new batch containing only the specified images in the order they were listed. |
|
||||
| `FL_ImageSlicer` | Divides an image into a grid of equally-sized rectangular slices based on x and y subdivision counts. Returns all slices as a batch of images, processing from left-to-right, top-to-bottom. |
|
||||
| `FL_LoadImage` | Loads images through a responsive drag-and-drop preview panel backed by the ComfyUI input directory, with Original, Fit, and Fill/crop resize modes plus optional connected width/height overrides. Existing absolute-path workflows remain supported, and transparent images are composited over white. |
|
||||
| `FL_PaddingRemover` | Detects and removes uniform padding from images by analyzing edge colors and their consistency across rows and columns. Supports CPU and GPU processing modes, configurable tolerance, minimum content size constraints, and additional manual trimming options with optional debug visualization. |
|
||||
| `FL_ReplaceColor` | Replaces pixels of a specific source color with a target color using Euclidean distance in RGB space and configurable tolerance. Supports morphological operations (expand/contract) to dilate or erode the replacement area using circular kernels. |
|
||||
| `FL_SaveAndDisplayImage` | Saves images to disk via ComfyUI's standard `SaveImage` path **and** passes the image batch through to downstream nodes, with the UI surface preview baked in for inline review. |
|
||||
| `FL_SaveImages` | Saves images to a custom folder structure with templated filenames and automatic index incrementing to avoid overwriting. Supports optional metadata saving as separate text files alongside images. |
|
||||
| `FL_SaveRGBAAnimatedWebP` | Saves animated WebP files from separate RGB and alpha channel image batches with configurable FPS, quality, and encoding method. Creates a preview with checkerboard background for transparency visualization and exports both the WebP animation and a ZIP file containing individual PNG frames with alpha. |
|
||||
| `FL_SaveWebM` | Exports image batches as WebM video files using VP9 codec with configurable FPS, quality, and optional alpha channel preservation. Uses imageio with ffmpeg backend and includes fallback handling for encoding errors. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-captioning-nodes">📝 Captioning Nodes</h2></summary>
|
||||
|
||||
> *Nodes for generating, saving, and manipulating image and video captions.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_CSVExtractor` | Extracts a specific column value from a randomly selected CSV row using a seed-based selection mechanism, supporting header skipping and returning the extracted text, row number, and full row data. Uses modulo-based pseudo-random selection for reproducible results. |
|
||||
| `FL_CaptionSaver_V2` | Advanced caption saver supporting both image tensor input and directory input modes with single or multiple caption options, optional mask image saving, and configurable downsize factors. Provides flexible caption assignment with automatic list extension/truncation and unique filename generation when overwrite is disabled. |
|
||||
| `FL_CaptionToCSV` | Scans an image directory and creates an in-memory CSV mapping image filenames to their corresponding caption text files. Returns CSV data as UTF-8 encoded bytes containing image_file and caption columns. |
|
||||
| `FL_ImageCaptionLayout` | Creates a visual grid layout combining images with their caption files from a directory, generating a single composite image with configurable rows, image sizes, padding, and caption boxes. Returns a PyTorch tensor suitable for display in ComfyUI. |
|
||||
| `FL_ImageCaptionLayoutPDF` | Generates both a PDF document and an image preview displaying a grid of images with captions from a directory, supporting pagination for multiple pages with configurable layout dimensions, orientation, and font settings. Uses ReportLab for PDF generation and PIL for image preview creation. |
|
||||
| `FL_ImageCaptionSaver` | Saves a batch of images and their associated caption text to a specified folder, sanitizing captions by removing special characters and supporting optional file overwrite prevention. Handles tensor-to-image conversion with automatic grayscale-to-RGB conversion and proper value normalization. |
|
||||
| `FL_LoadCSV` | Loads a CSV file from disk and returns both the raw binary data and row count for use in other nodes. Validates file existence, extension, and provides modification time tracking for cache invalidation. |
|
||||
| `FL_MirrorAndAppendCaptions` | Processes images in a directory by appending frame numbers to captions and optionally creating horizontally mirrored duplicates with sequential frame numbering. Supports prepending or appending custom text to captions and saves both original and mirrored versions back to the directory. |
|
||||
| `FL_OllamaCaptioner` | Generates image captions using Ollama LLM by encoding images as base64 and sending them to a local Ollama server for description generation. Saves images with either AI-generated or default captions, supporting custom model selection and URL configuration. |
|
||||
| `FL_SaveCSV` | Writes CSV data (received as bytes) to a specified file path, automatically creating output directories and ensuring proper .csv extension. Provides file writing with error handling and console logging for save confirmation. |
|
||||
| `FL_VideoCaptionSaver` | Converts a sequence of image tensors into a video file (MP4 or AVI) using OpenCV with configurable FPS and quality settings, while saving an associated caption text file. Handles RGB-to-BGR conversion, frame processing with progress tracking, and supports overwrite prevention. |
|
||||
| `FL_WordFrequencyGraph` | Analyzes all text files in a directory to generate a horizontal bar graph visualization of word frequency statistics with customizable color schemes, word filtering options, and configurable output dimensions. Excludes common stop words and produces a dark-themed matplotlib graph as a PyTorch tensor. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-vfx-nodes">✨ VFX Nodes</h2></summary>
|
||||
|
||||
> *Nodes for applying visual effects to images.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_Ascii` | Converts images to ASCII art by sampling pixels at regular intervals and mapping them to custom character sets, with options for sequential or brightness-based character selection, configurable spacing, font, and font size. |
|
||||
| `FL_DepthBlur` | Depth-of-field blur driven by a depth map: pre-computes N kernels (gaussian / box / disc), builds a per-pixel blur-amount map from focal depth and falloff curve (linear / quadratic / smooth / exponential), then lerps between layers. Supports independent near/far strength, depth invert/remap/smoothing, bokeh brightness boost, optional protect mask, and on-node preview. |
|
||||
| `FL_Dither` | Applies various dithering algorithms (Floyd-Steinberg, Random, Ordered, Bayer) to reduce images to a specified number of colors, creating retro-style color quantization effects with configurable color palette sizes from 2-256 colors. |
|
||||
| `FL_Glitch` | Creates digital glitch effects using the glitch_this library with controllable glitch intensity and optional RGB color channel offset, applying effects both horizontally and vertically by rotating the image 90 degrees during processing. |
|
||||
| `FL_HalftonePattern` | Generates halftone dot patterns by converting images to grayscale and creating circular dots of varying sizes based on region brightness, with configurable dot size and spacing parameters for newspaper-print style effects. |
|
||||
| `FL_HexagonalPattern` | Creates mosaic-style images by tiling hexagonal crops of the input with configurable shadow effects, background color, rotation, and spacing using PIL's polygon drawing capabilities. |
|
||||
| `FL_ImageCollage` | Generates photomosaics by tiling a template image across the base image, colorizing each tile based on the average color of the corresponding base region, with configurable tile size and spacing. |
|
||||
| `FL_ImagePixelator` | Applies pixelation effects with morphological gradient edge detection using Kornia, featuring configurable scale factor, kernel size, and sine-wave modulation for animated pixelation intensity across image sequences. |
|
||||
| `FL_InfiniteZoom` | Produces infinite zoom/droste effects using OpenGL shaders with GLFW, supporting configurable scale, mirror/swirl parameters, iteration count, and animation speed to create recursive visual patterns. |
|
||||
| `FL_PaperDrawn` | Simulates hand-drawn sketch effects using OpenGL fragment shaders that apply directional gradient analysis with configurable angle count, sampling density, line width, and vignette strength to create pencil-drawn aesthetics. |
|
||||
| `FL_PixelArtShader` | Creates pixel art effects by downsampling images with nearest-neighbor interpolation, applying color quantization, and optionally mapping to extracted color palettes using K-means clustering with mask support. |
|
||||
| `FL_PixelSort` | Implements pixel sorting effects based on HSV saturation values, sorting pixels within detected edge intervals either horizontally or vertically with configurable threshold, smoothing, and rotation parameters. |
|
||||
| `FL_RetroEffect` | Creates vintage CRT monitor effects by applying RGB color channel offset, scanline overlay, radial vignette with gaussian blur, and random noise blending using PIL image processing techniques. |
|
||||
| `FL_Ripple` | Generates radial ripple/wave distortion effects using sinusoidal displacement mapping based on distance from a configurable center point, with adjustable amplitude, frequency, phase, and optional sine-wave modulation. |
|
||||
| `FL_Shadertoy` | Renders Shadertoy-compatible GLSL shaders using OpenGL with support for up to 4 input image channels, iResolution/iTime/iFrame uniforms, and configurable frame count/FPS for shader-based generative effects. |
|
||||
| `FL_TextOverlayNode` | Overlays customizable text on images with support for TrueType fonts (system or local), configurable RGB color, percentage-based positioning, multiple anchor points (9 positions), and multiline text rendering using PIL ImageDraw. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-utility-nodes">🛠️ Utility Nodes</h2></summary>
|
||||
|
||||
> *General utility nodes for various tasks.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_ClipScanner` | Tokenizes text prompts using OpenCLIP tokenizers for different model types (SDXL ViT-G/14, SD 1.5 ViT-L/14, FLUX ViT-L/14) and displays detailed token analysis including token count and decoded output. |
|
||||
| `FL_DirectoryCrawl` | Recursively crawls directories to batch-load images or text files with progress tracking, returning images as both batched tensors (padded) and individual image lists for flexible processing. |
|
||||
| `FL_Float` | Outputs a float value rounded to 2 decimal places, serving as a simple float constant provider for workflows. |
|
||||
| `FL_FloatToInt` | Converts a FLOAT to an INT with a chosen rounding mode: `round` (banker's), `floor`, `ceil`, or `truncate`. |
|
||||
| `FL_IntToFloat` | Converts an INT to a FLOAT — a trivial type bridge for chaining numeric nodes. |
|
||||
| `FL_InpaintCrop` | Crops images based on mask bounding boxes with multiple sizing modes (free, forced, range, long/short side) including context expansion, optional hole filling (CPU/GPU), and generates stitch data for recomposition. |
|
||||
| `FL_Inpaint_Stitch` | Stitches inpainted images back into their original positions using crop data from FL_InpaintCrop, handling rescaling and alpha compositing to blend processed regions seamlessly. |
|
||||
| `FL_JS` | Executes custom JavaScript code in response to ComfyUI workflow events (before_queued, after_queued, executing, executed, etc.) for client-side scripting integration. |
|
||||
| `FL_Math` | Evaluates mathematical expressions with up to three variables (A, B, C) using safe AST parsing, supporting basic arithmetic operators (add, subtract, multiply, divide, power) and returning both integer and float results. |
|
||||
| `FL_ModelInspector` | Inspects PyTorch checkpoint files (.ckpt, .safetensors) and extracts comprehensive information including parameter count, layer shapes, architecture analysis, and model structure with support for various loading methods and fallback mechanisms. |
|
||||
| `FL_Padding` | Adds customizable padding to images on all four sides with configurable RGB color, optional alpha channel, and proper tensor handling for batched processing. |
|
||||
| `FL_PasteByMask` | Pastes images onto base images at mask locations with advanced features including multiple resize behaviors (resize, keep_ratio_fill/fit, source_size), blend modes (normal, multiply, screen, overlay, etc.), feathering, and mask mapping support for batch processing. |
|
||||
| `FL_PasteOnCanvas` | Pastes and centers masked images onto a canvas of specified dimensions with optional background color/image, supporting multiple resize algorithms (bilinear, nearest, bicubic, lanczos), padding, and alpha channel handling. |
|
||||
| `FL_PathTypeChecker` | Analyzes file path strings to determine their type (absolute, relative, UNC, URL-like, drive-specific) with validation and classification logic. |
|
||||
| `FL_RandomNumber` | Generates random integer and float values within specified ranges using seed-based reproducibility, with automatic seed generation when seed is 0. |
|
||||
| `FL_RandomShapeGenerator` | Generates batches of training-friendly synthetic images with random shapes (circles, rectangles, triangles, ellipses, lines, polygons, geometric_only) in configurable color modes (random / vibrant / pastel / monochrome / warm / cool / grayscale), with opacity range and background color. Outputs both image and latent. |
|
||||
| `FL_SDUltimate_Slices` | Calculates slice dimensions for SD Ultimate upscaling by dividing images into configurable grid patterns (1x1 to 4x4) with multiplier support for slice size adjustment. |
|
||||
| `FL_SeparateMaskComponents` | Separates multi-component masks into individual connected components using OpenCV's connected component labeling, returning separated masks and a mapping tensor for batch tracking. |
|
||||
| `FL_ShowText` | Displays a connected STRING input on the node via a styled DOM widget (header, length badge, monospace body, wheel-to-scroll capture) and passes the same string through to the output. |
|
||||
| `FL_StringToLoraName` | Bridges a free-form STRING (e.g. dynamic LoRA names generated upstream) to the canonical `loras` dropdown type via `folder_paths.get_filename_list("loras")`, so downstream LoRA-loader nodes accept it. |
|
||||
| `FL_Switch` | Implements lazy-evaluated boolean switch that evaluates and returns either on_true or on_false input based on switch value, enabling conditional workflow execution without evaluating unused branches. |
|
||||
| `FL_Switch_Big` | Implements lazy-evaluated multi-case switch logic (5 cases + default) by comparing a switch condition against case strings and evaluating only the matched input path for conditional workflow branching. |
|
||||
| `FL_SystemCheck` | Gathers and exposes system information including Python version, OS, CPU, RAM, GPU, library versions (PyTorch, transformers, etc.), and environment variables via an HTTP endpoint. |
|
||||
| `FL_UnloadAllModels` | Triggers ComfyUI's `unload_all_models()`, then runs `gc.collect()` + `torch.cuda.empty_cache()` + `torch.cuda.ipc_collect()` to aggressively free VRAM between heavy workflow stages. |
|
||||
| `FL_UnloadModel` | Removes models from memory and clears CUDA cache by unloading specified models from ComfyUI's model management system, with passthrough capability for workflow continuity. |
|
||||
| `FL_VideoCropMask` | Crops video frames based on mask bounding boxes with temporal smoothing, maintaining aspect ratio and optionally applying padding, returning cropped video, mask, and crop data for each frame. |
|
||||
| `FL_VideoRecompose` | Replaces cropped regions back into original video frames using crop data from FL_VideoCropMask, resizing processed frames to match original crop dimensions. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-ksamplers-nodes">🎲 KSamplers Nodes</h2></summary>
|
||||
|
||||
> *Nodes for sampling in the generation process.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_KSamplerXYZPlot` | Grid comparison sampler that generates parameter sweep visualizations by iterating over X/Y axis values (steps, CFG, denoise, sampler, or scheduler) with optional Z-axis stacking (vertical/horizontal), producing labeled image grids with axes and parameter annotations for systematic parameter testing. |
|
||||
| `FL_KsamplerBasic` | Simplified sampler node that accepts either latent or image input (with VAE encoding), performs standard KSampler operations with configurable steps/CFG/sampler/scheduler/denoise parameters, and outputs both latent and decoded image results. |
|
||||
| `FL_KsamplerContextWindow` | Video sampler with automatic Wan/LTX/MiniMax H3 frame conversion, compact window/overlap controls, expandable Advanced settings, and model/window progress diagnostics. Preserves the clip duration and native video/audio latents. |
|
||||
| `FL_KsamplerPlus` | Advanced tiled sampler that divides latent or image inputs into overlapping grid slices (configurable X/Y slices and overlap), processes each tile with optional region-specific conditioning (including ControlNet cropping), and blends results using cosine-weighted masks to eliminate seams. |
|
||||
| `FL_KsamplerPlusV2` | Enhanced version of FL_KsamplerPlus with additional features including user-configurable conditioning strength multiplier, complete conditioning split across tiles (including pooled outputs and tensor-based conditioning), batch processing support for sliced conditioning, and optional debug information output. |
|
||||
| `FL_KsamplerSEG` | Cascade-paint segmented sampler that walks per-region soft masks from `FL_KsamplerSEG_Regions` in a corner-anchored nearest-neighbor order, sampling each region against the in-progress canvas so the model itself does seam blending in feature space. Configurable cascade start corner. |
|
||||
| `FL_KsamplerSEG_Captioner` | Caption each `SEG_REGIONS` region's padded crop via OpenRouter vision models (GPT-4o-mini, Gemini Flash 1.5, Claude Haiku 4.5) with bounded parallel requests, streaming thumbnails+captions back to an on-node DOM widget as they arrive. Output captions feed the SEG encoder. |
|
||||
| `FL_KsamplerSEG_Encoder` | Turns per-region captions (from `FL_KsamplerSEG_Captioner`) into per-region CLIP CONDITIONING. Falls back to a shared default positive/negative for regions with no caption. CLIP encodes are cached by exact text to avoid duplicate work. |
|
||||
| `FL_KsamplerSEG_Regions` | Tessellates an image into N overlapping, feathered Voronoi regions (optional Lloyd relaxation, safe-zone subtraction). Returns soft masks where the SAME mask serves as both the noise mask and the composite blend mask, so coverage is guaranteed and the alpha math normalizes to ~1.0 everywhere. |
|
||||
| `FL_KsamplerSigma` | Custom sigma-schedule sampler that lets you draw a sigma curve directly on the node and feeds it into a standard KSampler call. Returns model/conditioning/latent/vae/image plus the SIGMAS tensor for inspection or reuse. |
|
||||
| `FL_KsamplerSettings` | Configuration node that outputs sampling parameters including aspect ratio dimensions (SD/XL/SVD presets with landscape/portrait rotation), batch size, two-pass step counts, CFG scales, denoise strength, scale factor, and sampler/scheduler selection. |
|
||||
| `FL_SamplerStrings` | String generator node with one boolean toggle per sampler reported by `comfy.samplers.KSampler.SAMPLERS` (dynamically populated so new ComfyUI samplers auto-appear) that outputs a comma-separated string of selected sampler names for batch processing workflows. |
|
||||
| `FL_SchedulerStrings` | String generator node with one boolean toggle per scheduler reported by `comfy.samplers.KSampler.SCHEDULERS` (dynamically populated) that outputs a comma-separated string of selected scheduler names for batch processing workflows. |
|
||||
|
||||
**Context Window KSampler:** New nodes use `auto`. Window length snaps down to the connected model's clip grid: Wan `4n+1`, LTX `8n+1`, H3 `17n+5`. Overlap snaps down independently to shared latent positions; H3 uses a bound valid at every token phase. The progress summary reports the resolved profile and latent counts; its tooltip includes requested values, overlap, and the extra causal anchor. Internal window spans can differ from standalone clip lengths. These settings do not resize the complete latent or change H3 shot timing.
|
||||
|
||||
Advanced contains the explicit frame profiles, raw `latent_frames`, schedules, blending, FreeNoise, causal handling, temporal axis, and conditioning options. H3 model-aware sampling currently supports `standard_static` with FreeNoise disabled. Batched ignores overlap; stride applies only to uniform schedules. Saved workflows keep their original `video_frames_4n_plus_1` or `latent_frames` interpretation until switched to Auto. Unknown models require an explicit profile or raw latent units.
|
||||
|
||||
H3 context sampling also requires the matching core support in `comfy/model_base.py` and `comfy/context_windows.py` for temporal positions, audio boundaries, and packed conditioning masks. Restart ComfyUI and reload the frontend after updating both core and Fill-Nodes.
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-pdf-nodes">📄 PDF Nodes</h2></summary>
|
||||
|
||||
> *Nodes for working with PDF files.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_BulkPDFLoader` | Loads all PDF files from a specified directory path and returns them as a list of PDF data objects containing file metadata (path, filename, page count) and binary content. Validates that PDFs exist and can be read, collecting errors for any failed loads. |
|
||||
| `FL_ImagesToPDF` | Converts ComfyUI image tensors into a single PDF document with configurable DPI and page size (A4/Letter/Legal), automatically scaling and centering each image to fit the page while maintaining aspect ratio. Returns an in-memory PDF data object. |
|
||||
| `FL_PDFEncryptor` | Encrypts a PDF with user and owner passwords using PyPDF2, supporting 40-bit or 128-bit encryption with granular permission controls (printing, copying, commenting, form filling, assembly, content extraction). Returns an encrypted in-memory PDF data object. |
|
||||
| `FL_PDFImageExtractor` | Extracts embedded images from PDF pages that meet minimum width/height thresholds, handling multiple compression formats (DCTDecode/JPEG, FlateDecode/PNG, JPXDecode/JPEG2000, LZWDecode). Returns extracted images as normalized PyTorch tensors in batch format [B, H, W, C]. |
|
||||
| `FL_PDFLoader` | Loads a single PDF file from a specified file path and returns it as a PDF data object containing the file path, page count, and binary content. Validates file existence and PDF format before loading. |
|
||||
| `FL_PDFMerger` | Merges two PDF data objects into a single combined PDF using PyPDF2's PdfMerger, appending the second PDF after the first. Returns the merged PDF as an in-memory data object with total page count. |
|
||||
| `FL_PDFSaver` | Saves a PDF data object to disk at a specified directory path with a given filename, automatically creating the output directory if needed and appending '.pdf' extension if not present. Returns the full output file path. |
|
||||
| `FL_PDFTextExtractor` | Extracts all text content from a PDF document by iterating through pages using PyPDF2's text extraction, combining text from all pages with double newlines as page separators. Returns the extracted text as a single string. |
|
||||
| `FL_PDFToImages` | Converts PDF pages to images using PyMuPDF (fitz) with configurable DPI resolution, supporting both single PDFs and lists of PDFs. Returns all pages as normalized PyTorch image tensors concatenated into a single batch. |
|
||||
| `FL_TextToPDF` | Generates a PDF document from plain text input using ReportLab canvas, with configurable page size, font size, margins, and title, automatically handling text wrapping and pagination. Returns an in-memory PDF data object with calculated page count. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-gpt-nodes">🤖 GPT Nodes</h2></summary>
|
||||
|
||||
> *Nodes for integrating with GPT and OpenAI models.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_Dalle3` | Generates images using OpenAI's DALL-E 3 model with asynchronous batch processing, caching results based on parameters to avoid redundant API calls, and returns image tensors along with revised prompts. Supports auto-save functionality with metadata JSON files and configurable retry logic for API failures. |
|
||||
| `FL_GPT_Image1` | Generates or edits images using OpenAI's gpt-image-1 model with support for batch generation, image editing with optional masks, and comprehensive error handling that creates error images with visual text feedback. Implements retry logic, multipart form-data for edits endpoint, and processes both base64 and URL-based image responses. |
|
||||
| `FL_GPT_Image1_ADV` | Advanced multi-input variant that generates multiple images concurrently using OpenAI's gpt-image-1 model by processing up to 100 prompts with individual image inputs for editing in parallel using async/await and thread pool execution. Each input slot can have its own prompt and optional image, with all outputs combined into a batched tensor and detailed per-call response logs. |
|
||||
| `FL_GPT_Text` | Makes synchronous API calls to OpenAI's GPT models (GPT-4, GPT-3.5-turbo) for text completion with full parameter control including temperature, top_p, and frequency/presence penalties. Optionally saves responses to file and falls back to environment variable for API key if not provided. |
|
||||
| `FL_GPT_Vision` | Batch processes images using OpenAI's GPT-4 Vision models to generate captions asynchronously with configurable batch sizes, supporting both input directories and image tensors, and saves captions to text files alongside images. Retrieves API key from environment variable OPENAI_API_KEY. |
|
||||
| `FL_SimpleGPTVision` | Sends a single image to OpenAI's GPT-4 Vision API to generate a text caption based on configurable system and user prompts, with built-in rate limiting retry logic using exponential backoff. Returns the generated caption as a string output. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-ai-nodes">🧠 AI Nodes</h2></summary>
|
||||
|
||||
> *Nodes that integrate with various AI models and services.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_Fal_Gemini_ImageEdit` | Edits up to 5 input images using Fal AI Gemini 2.5 Flash Image Edit API with multimodal prompt support, async processing, and automatic image URL generation. Returns edited images with URLs and description. |
|
||||
| `FL_Fal_GPTImage2_Edit` | Edits/composes a variable number of reference images (dynamic sockets driven by `image_count`) against a shared prompt via Fal AI's `openai/gpt-image-2/edit` endpoint. Configurable size presets / custom dimensions, quality (low/medium/high), num_images (1-4), and output format (png/jpeg/webp). |
|
||||
| `FL_Fal_Kling_AIAvatar` | Generates AI avatar videos using Fal AI Kling Video API by combining input image with audio tensor or image sequence, with optional text prompt refinement. Returns extracted video frames and original audio. |
|
||||
| `FL_Fal_Kontext` | Edits images using Fal AI Flux Pro Kontext API with multi-image input support (batching), asynchronous parallel processing, and configurable parameters including aspect ratio and safety tolerance. Returns batched edited images from multiple API calls. |
|
||||
| `FL_Fal_Pixverse` | Generates videos from single image using Fal AI Pixverse v5 Image-to-Video API with style presets, camera movements, and parallel batch processing (up to 5). Returns extracted frames from generated videos as tensors. |
|
||||
| `FL_Fal_Pixverse_LipSync` | Generates lip-sync videos using Fal AI Pixverse LipSync API supporting both audio input and text-to-speech modes with voice selection, processing video frames tensor with audio synchronization. Returns lip-synced video frames and audio. |
|
||||
| `FL_Fal_Pixverse_Transition` | Creates transition videos between two images using Fal AI Pixverse v5 Transition API with configurable resolution, aspect ratio, and duration. Returns extracted frames from generated transition videos. |
|
||||
| `FL_Fal_Seedance2_ReferenceToVideo` | Generates native ComfyUI VIDEO output through Fal's `bytedance/seedance-2.0/reference-to-video` endpoint. Supports prompt-only generation or Autogrow groups of up to 9 images, 3 videos, and 3 audio clips, with clickable `@Image1`, `@Video1`, and `@Audio1` prompt tags plus an on-node queue and result dashboard. |
|
||||
| `FL_Fal_Seedance_i2v` | Generates videos from single image using Fal AI ByteDance Seedance v1 Pro Image-to-Video API with support for multi-segment prompts using [cut] markers, configurable resolution, duration, and camera settings. Returns all extracted frames from generated video. |
|
||||
| `FL_Fal_Seedream_Edit` | Edits up to 10 images using Fal AI ByteDance Seedream v4 Edit API with custom or preset image sizes, auto-scaling to meet minimum dimensions while preserving aspect ratio. Returns edited images with URLs and seed information. |
|
||||
| `FL_Fal_SeedVR_Upscale` | Upscales a single image via Fal AI's SeedVR endpoint with a configurable upscale factor (1.0×–10.0×) and seed. Uploads the tensor to fal.media, polls for completion, and returns the upscaled image plus its URL and final dimensions. |
|
||||
| `FL_Fal_Sora` | Generates videos using Fal AI Sora 2 API supporting both text-to-video and image-to-video modes with optional OpenAI API key pass-through, automatic endpoint selection based on PRO mode, and audio extraction. Returns video frames, audio waveform, video URL, and status message. |
|
||||
| `FL_GeminiImageEditor` | Generates or edits images using Google Gemini image models with support for up to 4 reference images, batch generation with parallel async processing, and optional square padding. Returns list of generated images with detailed API response logs. |
|
||||
| `FL_GeminiImageGenADV` | Advanced multi-input image generation using Google Gemini with dynamic input count (1-100), async parallel batch processing, and per-input prompt/image pairs. Returns list of generated images based on variable number of input slots. |
|
||||
| `FL_GeminiTextAPI` | Generates text from a prompt and optional AUDIO, IMAGE, and native VIDEO inputs through Gemini's Interactions API. Supports mixed media, image/audio batches, system instructions, and a model override. See [media inputs and music captioning](docs/gemini_music_captioning.md). |
|
||||
| `FL_GeminiVideoCaptioner` | Generates detailed captions for videos or image sequences using current Gemini multimodal models, with Files API upload for video inputs and a sampled-frame output. |
|
||||
| `FL_Hedra_API` | Generates videos from image, audio file, and text prompt using Hedra API with configurable aspect ratio and resolution, automatic polling for generation completion, and frame extraction from downloaded video. Returns video frames tensor with processing logs. |
|
||||
| `FL_PixVerseAPI` | Generates videos from images using PixVerse API with support for standard image-to-video and transition modes, parallel batch processing with configurable seeds, and automatic frame extraction from generated MP4 videos. Returns up to 5 batches of extracted frames as tensors. |
|
||||
| `FL_RunwayAct2` | Generates character performance videos using RunwayML Act Two API from input character images/videos and reference videos, with controls for body movement and expression intensity. Returns extracted video frames as tensors. |
|
||||
| `FL_RunwayImageAPI` | Generates images using RunwayML Gen4 Image API with support for up to 3 reference images with custom tags, configurable generation parameters, and automatic polling for task completion. Returns generated image tensor and detailed status logs. |
|
||||
| `FL_Veo3VideoGen` | Generates videos using Google Vertex AI Veo 3.0 models with service account authentication, optional reference image input, configurable aspect ratio/resolution, and automatic polling with frame extraction. Returns extracted video frames, video path, and processing logs. |
|
||||
| `FL_VertexGemini25FlashImage` | Generates images using Google Vertex AI Gemini 2.5 Flash Image model with service account authentication, support for up to 3 reference images, and parallel batch generation. Returns batch tensor of generated images with detailed processing logs. |
|
||||
|
||||
For `FL_Fal_Seedance2_ReferenceToVideo`, set `FAL_KEY` in the ComfyUI environment or use the advanced `fal_api_key` override. Connected references are uploaded in numbered socket order; reference audio requires at least one image or video. The node returns the generated native `VIDEO`, remote URL, Fal-provided seed, and request ID.
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-audio-nodes">🔊 Audio Nodes</h2></summary>
|
||||
|
||||
> *Nodes for audio analysis, processing, and audio-reactive visual effects.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_Audio_BPM_Analyzer` | Analyzes audio using librosa to detect BPM and beat positions, with options for two BPM calculation methods (beat_intervals or onset_strength), beat offset adjustment, and automatic beat filling to cover the entire audio duration. Outputs beat positions as JSON with visualization. |
|
||||
| `FL_Audio_Beat_Prompt_Envelope` | Turns exact detected beats into attack/hold/release prompt-mask pulses for compatible FL diffusion video nodes. Supports beat stride, phase, response curve, and on-beat dominance above normal conditioning weight. |
|
||||
| `FL_Audio_Beat_Prompt_Schedule` | Compact node with a full-screen audio/prompt sequencer, Beat This analysis, waveform-aligned trimming, frame-native prompt clips, local Whisper transcription or LRC/SRT import, an editable timed-lyrics lane, Beat Writer lyric context, and three live reactive-envelope editors. Outputs the schedule, frame count, cropped audio, BPM, three typed audio envelopes, and one prompt-envelope set. |
|
||||
| `FL_Audio_Beat_Visualizer` | Generates video frames that visualize beat patterns by either alternating between black/white on beat changes or cycling through provided images, with configurable frame dimensions and starting color. |
|
||||
| `FL_AudioFrameCalculator` | Given an AUDIO and a target FPS, returns the integer frame count needed to cover the audio's duration (`ceil(duration * fps)`). Useful upstream of video-generation nodes that need an exact frame budget. |
|
||||
| `FL_Audio_Crop` | Crops audio waveforms to specified start and end times using MM:SS or seconds format, with automatic clamping to valid audio boundaries. |
|
||||
| `FL_Audio_Drum_Detector` | Detects kicks, snares, and hi-hats from audio using librosa onset detection with frequency band analysis (kicks: 30-300Hz, snares: 150-400Hz + 4-10kHz noise, hi-hats: 6kHz+) and adjustable sensitivity thresholds. |
|
||||
| `FL_Audio_Envelope_Prompt` | Maps an existing FL audio envelope to diffusion prompt-mask strength, with threshold, response curve, inversion, and floor/peak controls. |
|
||||
| `FL_Audio_Envelope_Visualizer` | Visualizes audio envelopes as grayscale video frames where pixel intensity corresponds to envelope values, with configurable intensity multiplier and optional color inversion. |
|
||||
| `FL_Audio_Music_Video_Sequencer` | Generates complete music video shot sequences from beat positions using pattern-based orchestration (A/B/C/D patterns with configurable beat counts), creating drift-free frame boundaries and detailed metadata for each shot including time, beat, and sample boundaries. |
|
||||
| `FL_Audio_Reactive_Brightness` | Multiplies frame pixel values by brightness factors calculated from audio envelopes, with configurable base brightness, intensity scaling, optional inversion, and output clamping. |
|
||||
| `FL_Audio_Reactive_Edge_Glow` | Applies audio-reactive edge detection using Sobel operators (or accepts pre-computed edge frames) and adds colored glow effects that pulse with envelope values, supporting multiple blend modes (add, screen, overlay) and glow colors. |
|
||||
| `FL_Audio_Reactive_Envelope` | Generates per-frame ADSR envelopes (Attack, Decay, Sustain, Release) from drum detection times (kicks, snares, hi-hats) with independent configurable envelope parameters for each drum type. |
|
||||
| `FL_Audio_Reactive_Saturation` | Adjusts color saturation by blending between grayscale (using Rec. 709 luma coefficients) and original frames based on envelope-modulated saturation values, supporting both saturation and desaturation effects. |
|
||||
| `FL_Audio_Reactive_Scale` | Applies audio-reactive zoom/scale effects to frames based on envelope values, with center-cropping or padding to maintain dimensions and configurable interpolation modes (bilinear, bicubic, nearest). |
|
||||
| `FL_Audio_Reactive_Speed` | Time-remaps frames using cumulative speed multipliers derived from audio envelopes, allowing for dynamic speed-up/slow-down effects with frame interpolation and optional envelope inversion. |
|
||||
| `FL_Audio_Segment_Extractor` | Extracts audio segments based on pre-analyzed beat positions from the BPM analyzer, supporting drift-free frame alignment by using cumulative time mapping and outputting precise frame counts for video synchronization. |
|
||||
| `FL_Audio_Separation` | Separates audio into four stems (bass, drums, other, vocals) using the Hybrid Demucs model from torchaudio, with chunked processing to manage memory usage and configurable overlap/fade parameters. |
|
||||
| `FL_Audio_Shot_Iterator` | Extracts individual shot metadata (frame counts, beat ranges, time boundaries) from music video sequence JSON by shot index, designed for iterative processing of video sequences. |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-experimental-nodes">🧪 Experimental Nodes</h2></summary>
|
||||
|
||||
> *Experimental nodes with various functionalities.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_BatchAlign` | Patches a model to enable batch-aligned generation by applying style alignment techniques through shared attention mechanisms and normalization layers. Uses adaptive instance normalization (AdaIN) on queries, keys, and values with configurable attention sharing and normalization strategies (group/layer/both). |
|
||||
| `FL_ColorPicker` | A simple utility node that accepts a hex color string input and passes it through as output, designed to facilitate color selection in workflows. |
|
||||
| `FL_GradGenerator` | Generates gradient images from a JSON-formatted color stops specification with position and color values. Supports RGB/HSV color modes and multiple interpolation curves (Linear, Ease In, Ease Out, Ease In-Out) with configurable dimensions. |
|
||||
| `FL_NFTGenerator` | Randomly selects images from a directory based on rarity percentages encoded in filenames (format: name-XXXper-val), using weighted probability selection with a configurable seed. Returns both the selected image and its corresponding mask file (or blank mask if not found). |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-prompting-nodes">💬 Prompting Nodes</h2></summary>
|
||||
|
||||
> *Nodes for generating and manipulating prompts.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_MadLibGenerator` | Generates text by replacing up to 5 different delimiter patterns in a template with randomly selected words from corresponding word lists. Uses per-list seeding with MD5 hashing to ensure reproducible but independent randomization for each delimiter. |
|
||||
| `FL_PromptBasic` | Concatenates three text strings (prepend, prompt, append) without adding any spaces, preserving user-specified formatting for prompt construction. |
|
||||
| `FL_PromptMulti` | Processes multi-line positive and negative prompts into parallel lists, automatically balancing counts by repeating the last prompt or using empty strings. Generates indexed names with configurable prefix and outputs as lists for batch processing. |
|
||||
| `FL_PromptSelector` | Selects a single prompt line by index from a multi-line prompt string, optionally prepending and appending text with automatic space insertion. |
|
||||
| `FL_PromptSelectorBasic` | Selects a single line from a multi-line prompt string by index, wrapping the index with modulo so it always loops within bounds. No prepend/append. |
|
||||
|
||||
MiniMax H3 timeline, beat-planning, sampling, upscale, and shot-assembly nodes now live in the dedicated [ComfyUI-FL-MiniMaxH3](https://github.com/filliptm/ComfyUI-FL-MiniMaxH3) pack. Their node IDs are unchanged, so existing workflows continue to load after installing the new pack.
|
||||
|
||||
For audio-reactive timing, choose or upload audio directly on `FL_Audio_Beat_Prompt_Schedule`, then connect its `prompt_schedule`, `audio`, and `prompt_envelopes` outputs to **FL MiniMax H3 Beat Shot Planner** from the dedicated pack. The three `FL_AUDIO_ENVELOPE` outputs can independently drive the visualizer, image-space effects, and VFX nodes. A connected `beat_positions` input remains an optional timing override for existing workflows:
|
||||
|
||||
```text
|
||||
[0 - 48 | fade_in=6 | fade_out=6]
|
||||
The subject slowly turns toward camera.
|
||||
|
||||
[48 - 96 | fade_out=6 | crossfade=12]
|
||||
The camera pushes forward on the beat.
|
||||
```
|
||||
|
||||
Frame ranges are zero-based and range ends are exclusive. `fade_in` and `fade_out` blend a scheduled prompt with the global prompt. `crossfade` belongs to the incoming prompt and blends it directly with a touching previous prompt; the two scheduled weights retain full conditioning coverage throughout the transition. Detected beats appear in their own marker lane and can be used as the editor's snap target without changing the stored frame timing. A connected prompt schedule overrides the manual timeline. Exact repeated prompts share one conditioning mask, so reuse prompts for recurring sections such as choruses.
|
||||
|
||||
The prompt schedule node loads its waveform as soon as an audio file is selected; queueing is not required. Use the source overview handles to trim, Play for continuous crop-aligned looping, Pause/Stop for transport control, and the main frame ruler to drag or resize prompt clips. The transport volume control is remembered per sequencer node and changes only browser preview loudness, never the audio output. Drag the shared center boundary between touching prompts to expand one while shortening the other; the pair stays contiguous and the boundary snaps to the beat grid. Right-click a beat-aligned position on the main timeline to set the audio In or Out point; prompt clips and envelopes are cropped and rebased with the audio so they stay aligned to the same musical moments. The Beat offset slider moves the working beat grid and beat-derived envelopes while the waveform, raw detections, transients, drums, and audio remain fixed. Analysis covers and caches the full source by its content hash and detector settings; crop, FPS, length, envelope, and offset edits reuse that analysis without rescanning.
|
||||
|
||||
Beat and downbeat timing comes from the open-source MIT-licensed [`beat-this==1.1.0`](https://github.com/CPJKU/beat_this) package and its `final0` checkpoint. The first analysis explicitly downloads the 77.3 MiB checkpoint to `ComfyUI/models/beat_this/beat_this-final0.ckpt`; the sequencer shows download, verification, load, and analysis state. The artifact is SHA-256 verified before use, never downloads in the background, and is released from GPU memory after each analysis. Beat This analyzes the full master mix it was trained for; the Analysis source selector affects only Librosa onset and drum-family reference markers. Cyan circles are the offset-adjustable beat grid, gold diamonds and alternating shading are downbeats/bar starts, short magenta ticks are raw model beats, short orange ticks are transient onsets, and rose ticks identify intervals more than 25% from the median. Connecting `beat_positions` remains authoritative and skips Beat This during queued node execution.
|
||||
|
||||
`Separate stems` is an explicit action and never runs automatically. It separates the full source once with Hybrid Demucs, caches bass/drums/other/vocals locally, switches analysis to the drums stem, and keeps the node's audio output on the original master crop. Jobs report progress, can be cancelled between chunks, and reuse valid cached stems.
|
||||
|
||||
`Transcribe lyrics` runs Whisper locally against the cached vocals stem when available, or the full mix otherwise. The selected pinned Whisper model downloads to `ComfyUI/models/whisper/` only when that button is clicked; transcripts are content-addressed and cached locally. The lyrics lane supports frame-level timing and text correction plus timed LRC/SRT import. When **Include timed lyrics in Beat Writer** is enabled, only the visible lyric text and timing are included as read-only writing context; audio is never sent to the Writer.
|
||||
|
||||
The lower sequencer inspector contains three fixed reactive-envelope slots. Each slot selects the beat grid, downbeat, raw beat, transient, kick, snare, or hi-hat source and provides stride, phase, attack, hold, release, curve, prompt floor, and prompt peak controls. Grayscale strips and current-frame tiles preview the normalized signal live at the audio playhead. Empty prompt text leaves a signal-only envelope for image-space effects; nonempty prompts are collected into `FL_PROMPT_ENVELOPE_SET` for the Beat Shot Planner.
|
||||
|
||||
All FL audio-reactive and VFX nodes use the dedicated `FL_AUDIO_ENVELOPE` socket instead of generic JSON strings. `FL_Audio_Reactive_Envelope` still provides independent kick, snare, and hi-hat ADSR sources, and `FL_Audio_Envelope_Prompt` remains available when a typed signal needs to become an individual prompt envelope outside the sequencer workflow.
|
||||
|
||||
Strict scheduling evaluates H3 once per unique active prompt on every sampling step. Start with a few broad sections, and disable cross-step caching or model compilation while validating a masked workflow. The semantic output keeps normal single-prompt sampling cost.
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-file-operation-nodes">📂 File Operation Nodes</h2></summary>
|
||||
|
||||
> *Nodes for file operations.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_ZipDirectory` | Creates a ZIP archive of an entire directory structure in memory using ZIP_DEFLATED compression, returning the raw bytes without writing to disk. |
|
||||
| `FL_ZipSave` | Compresses a directory into a ZIP file and saves it to a specified output location with automatic filename collision handling via timestamp suffixing. |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-google-drive-nodes">☁️ Google Drive Nodes</h2></summary>
|
||||
|
||||
> *Nodes for Google Cloud services integration.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_GoogleCloudStorage` | Uploads images or compiled videos to Google Cloud Storage buckets using service account credentials. Supports batch image uploads, video compilation with configurable codecs/FPS, public/private access control, and custom metadata attachment. |
|
||||
| `FL_GoogleDriveDownloader` | Downloads files from Google Drive using share links, automatically extracts ZIP archives, and manages output in organized directory structures with cleanup of temporary files. |
|
||||
| `FL_GoogleDriveImageDownloader` | Downloads images from Google Drive share links with optional local caching system using MD5-hashed index. Converts downloaded images to RGB tensors normalized to [0,1] range with configurable cache behavior. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-api-tool-nodes">🔌 API Tool Nodes</h2></summary>
|
||||
|
||||
> *Nodes for API interactions.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_API_Base64_ImageLoader` | Decodes base64-encoded images (with automatic data URL prefix removal) and converts them to ComfyUI image tensors. Supports optional resizing with aspect ratio preservation and returns image tensor with dimensions, job_id, user_id, and category. |
|
||||
| `FL_API_ImageSaver` | Saves images to a categorized directory structure (base/category/user_id/job_id.format) with support for PNG, JPG, JPEG, and WebP formats and configurable quality settings. Returns full path, job_id, user_id, and category for API workflows. |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-hugging-face-nodes">🤗 Hugging Face Nodes</h2></summary>
|
||||
|
||||
> *Nodes for integrating with Hugging Face.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_HFDatasetDownloader` | Downloads Hugging Face repositories (datasets, models, or spaces) using snapshot_download with configurable parallel workers and local directory specification. |
|
||||
| `FL_HFHubModelUploader` | Uploads models and assets to Hugging Face Hub with automatic README generation, model card header support, and organized file structure. Handles images, ZIP files, and large model files with threaded progress tracking and retry logic. |
|
||||
| `FL_HF_Character` | Uploads character-related assets (LoRA, datasets, captions, CSVs) to Hugging Face Hub using a structured path format (studio/project/character) for organized character library management. |
|
||||
| `FL_HF_UploaderAbsolute` | Uploads various file types (LoRA files, ZIP datasets, images, PDFs, CSVs) to Hugging Face repositories at specified paths with progress tracking. Supports repository creation and uses environment variable HUGGINGFACE_API_KEY for authentication. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-loader-nodes">⏬ Loader Nodes</h2></summary>
|
||||
|
||||
> *Nodes for loading various resources.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
The MiniMax Music 3 loaders now live in the dedicated `ComfyUI-FL-MiniMaxMusic3` pack. Their node IDs are unchanged, so existing workflows remain compatible when that pack is installed.
|
||||
| `FL_NodeLoader` | A pass-through node that accepts and returns a TRIGGER type, designed for node dependency management in workflows. |
|
||||
| `FL_NodePackLoader` | A pass-through node with IS_CHANGED set to float('NaN') to ensure it always processes when triggered, useful for forcing workflow execution. |
|
||||
| `FL_UpscaleModel` | Upscales images in configurable batches using loaded upscale models with precision control (auto/32/16/bfloat16), optional downscaling by factor, multiple rescale methods (nearest-exact, bilinear, area, bicubic, lanczos), and TQDM progress tracking. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-discord-nodes">💬 Discord Nodes</h2></summary>
|
||||
|
||||
> *Nodes for Discord integration.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_SendToDiscordWebhook` | Sends single images or video compilations to Discord via webhooks with configurable bot username, custom messages, user mentions (via Discord user IDs), FPS control for videos, and optional local file retention. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-work-in-progress-nodes">🚧 Work-in-Progress Nodes</h2></summary>
|
||||
|
||||
> *Nodes that are still in development.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_AnimatedShapePatterns` | Generates batches of animated geometric shape frames using 6 pattern algorithms (grid_wave, grid_ripple, circle_array, orbital, spiral, lissajous) with configurable shape, frame count, dimensions, color, and motion parameters. Outputs both IMAGE and MASK. |
|
||||
| `FL_FractalKSampler` | Experimental recursive K-sampler that generates images through fractal upscaling and slicing. Upscales latents, divides into 4 quadrants, recursively samples each with reduced denoise, and composites results with configurable blend factors. |
|
||||
| `FL_LTXVMaskedICLoRAGuide` | LTXV-aware IC-LoRA guide node that snaps a guide image+mask to the model's latent cadence at a chosen frame index, exposing `guide_strength` and `denoise_strength` for controlling the IC-LoRA conditioning. Uses the `_INPAINT_GREEN` LTXV inpaint marker. |
|
||||
| `FL_PathAnimator` | Interactive node that pops a modal path-editor UI for drawing animated motion paths, with selectable interpolation easing (linear, ease-in, ease-out, ease-in-out) and per-frame point output. Integrates with `FL_ImagePicker`-style pause-and-pick session storage. |
|
||||
| `FL_QwenImageEditStrength` | Per-image strength variant of `TextEncodeQwenImageEditPlus` that lets you control the weight of each of up to 3 reference images (-10.0 to 10.0) and pick the latent combination method (weighted_sum / average / maximum / minimum / multiply). |
|
||||
| `FL_TimeLine` | 🚧 In-progress timeline scaffolding for video animation with IPAdapter preset selection, dimension control, interpolation modes, and frame/FPS settings. Includes an API endpoint for timeline data but currently returns the input MODEL unchanged. |
|
||||
| `FL_WF_Agent` | Workflow agent node that executes a node scanner to generate cached node definitions for AI-assisted workflow manipulation. Runs scanner as subprocess and provides feedback on cache file creation and status. |
|
||||
| `FL_WanFirstLastFrameToVideo` | Enhanced WAN (Warp and Noise) implementation for video generation from first and last frames with advanced timing control. Supports custom keyframe positioning, temporal interpolation curves (linear, ease, bounce), regional masking, and CLIP vision conditioning. |
|
||||
| `FL_WanVaceToVideoMultiRef` | WAN VACE video-generation node that accepts a control_video + control_masks + multi-frame reference_video (instead of a single reference image), encoding up to N reference frames via VAE and prepending them. Returns positive/negative conditioning, latent, and trim_latent count. |
|
||||
| `FL_WanVideoBlender` | Cross-fades two WAN-generated video clips over a configurable overlap-frame count. Auto-resizes the second clip to match the first. |
|
||||
| `FL_WanVideoBlender3Way` | Three-clip cross-fade variant of `FL_WanVideoBlender` for `Input → Continuation → End` workflows. Resizes the second and third clips to match the first. |
|
||||
| `FL_WanVideoContinue` | Builds a continuation-ready frame buffer by placing overlap frames from the end of an input video at the start and (optionally) overlap frames from the start of an end video at the end, with empty-frame fill level and optional control images / inpaint mask. |
|
||||
| `FL_WanVideoContinuationBlender` | Specialized blender for clips produced by `FL_WanVideoContinue` — auto-strips the embedded overlap frames before blending so frames aren't duplicated. Use this *after* the continuation generation step. |
|
||||
| `FL_ZImageControlNetPatch` | Loads a Z-Image / Lumina ControlNet patch with auto-config support and exposes `n_control_layers`, `additional_in_dim`, and `refiner_control` for manual tuning. Marked `EXPERIMENTAL = True`. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-game-nodes">🎮 Game Nodes</h2></summary>
|
||||
|
||||
> *Nodes implementing games.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_BulletHellGame` | ⚠️ Placeholder stub — reserved for future bullet-hell game functionality. Currently a no-op. |
|
||||
| `FL_TetrisGame` | ⚠️ Placeholder stub — reserved for future Tetris game functionality. Currently a no-op. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-video-nodes">🎬 Video Nodes</h2></summary>
|
||||
|
||||
> *Nodes for video processing and frame interpolation.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_FILM` | Frame interpolation using FILM (Frame Interpolation for Large Motion) from Google Research. Downloads TorchScript model from HuggingFace on first use, generates intermediate frames with configurable multiplier, and uses timestep-based interpolation for smooth motion. |
|
||||
| `FL_ProResVideo` | Converts image batches to ProRes 4444 video files using FFmpeg with high-quality encoding (yuva444p10le pixel format, 8000 bits_per_mb). Automatically handles temporary file creation, unique naming with timestamp collision avoidance, and cleanup. |
|
||||
| `FL_RIFE` | Frame interpolation using RIFE (Real-Time Intermediate Flow Estimation) v4.7 architecture. Downloads model weights from HuggingFace, supports multiple checkpoints (rife47, rife49), configurable multiplier, optional ensemble mode for quality, and multi-scale flow estimation. |
|
||||
| `FL_VideoBatchSplitter` | Splits a video batch into exactly N equal-sized sub-batches with strict frame count validation. Returns up to 20 separate batch outputs with configurable frames per batch, requiring exact division with no remainders. |
|
||||
| `FL_VideoCadence` | Analyzes image batches for scene changes and returns comma-separated frame counts per scene. Uses configurable detection methods (intensity, histogram, or hybrid) with GPU acceleration support, downsampling optimization, and minimum scene length enforcement. |
|
||||
| `FL_VideoCadenceCompile` | Loads videos from a directory, crops each video from the middle to match specified frame counts from a cadence list, and concatenates all frames into a single image batch. Validates frame counts and dimensions with error handling. |
|
||||
| `FL_VideoCrossfade` | Blends two video sequences with configurable crossfade duration using multiple blend modes (normal, multiply, screen, overlay, soft_light, add, subtract). Resizes both inputs to target dimensions and generates smooth transitions between sequences. |
|
||||
| `FL_VideoCut` | Detects scene cuts in image batches and exports each scene as a separate MP4 file using parallel FFmpeg processing. Features optimized scene detection (intensity/histogram/hybrid), GPU acceleration, downsample optimization, and direct frame piping to FFmpeg for performance. |
|
||||
| `FL_VideoPicker` | Splits a decoded video mosaic into an X×Y grid, presents synchronized looping previews for multi-selection, and emits each selected slice as a separate video-list item. Intended for zero-overlap tiled KSampler outputs; route shared audio around the picker. |
|
||||
| `FL_VideoTrim` | Trims specified numbers of frames from the start and end of an image batch, returning the trimmed video along with separate outputs for removed start and end frames. |
|
||||
|
||||
### 📷 Screenshots & Examples
|
||||
|
||||
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
|
||||
<details>
|
||||
<summary><h2 id="-kartel-nodes">📋 Kartel Nodes</h2></summary>
|
||||
|
||||
> *Nodes for Kartel job-pipeline integration.*
|
||||
|
||||
| Node | Description |
|
||||
|------|-------------|
|
||||
| `FL_KartelJobInput` | Decodes a Kartel job payload (job_id, user_id, user_email, app_name, callback_url, params_json) into 25 typed outputs — five job-metadata strings plus four each of strings / ints / floats / bools / images. Image fields are downloaded from `image_url_N` keys inside `params_json`. |
|
||||
| `FL_KartelJobOutput` | POSTs an image batch back to a Kartel callback URL as multipart PNGs with a JSON metadata field (job_id, user_id, user_email, app_name). Skips the POST silently when no callback URL is provided or the input is a 1×1 placeholder. |
|
||||
|
||||
</details>
|
||||
|
||||
---
|
||||
# ComfyUI_Fill-Nodes
|
||||
|
||||
Image randomizer from directory, Image Captioning saver.
|
||||
|
||||
Image randomizer: - A load image directory node that allows you to pull images either in sequence (Per que render) or at random (also per que render)
|
||||
-
|
||||
## Video
|
||||
|
||||
|
||||
https://github.com/filliptm/ComfyUI_Fill-Nodes/assets/55672949/8d222109-cda1-40f8-9558-6f8136e07617
|
||||
|
||||
It uses a dummy int value that you attach a seed to to enure that it will continue to pull new images from your directory even if the seed is fixed. In order to do this right click the node and turn the run trigger
|
||||
into an input and connect a seed generator of your choice set to random.
|
||||
|
||||
interesting uses for this
|
||||
|
||||
|
||||
-loading up a directory and letting it cycle through all your images in order
|
||||
|
||||
-connecting this node to something like IPAdapter, while being set to random, allowing you to cycle through styles via images
|
||||
|
||||
-batch processing of any kind on large amounts of images
|
||||
|
||||
Image Captioning saver: - takes an input image (single or batch) and saves a matching .txt file with the image with desired captioning.
|
||||
-
|
||||
Both files will be over written for continuous experimentation. Required to have an output attached for monitoring. Will overwrite images and text on each run. Built this node to save Lora captions from my Dataset Creator
|
||||
|
||||
|
||||
|
||||
Dimension Display: - Simply shows the dimension of an image in a string for monitoring. No need for INTS.
|
||||
-
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Pixelator: - Custom effect that bit reduces an image and makes it black and white. See examples
|
||||
-
|
||||
|
||||
current implementation requires you to break batches into a list and back into a batch if you want to use it on video. for VRAM management.
|
||||
|
||||
|
||||
|
||||
|
||||
Audio Tools (WIP): - Load audio, scans for BPM, crops audio to desired bars and duration
|
||||
-
|
||||
|
||||
other nodes that are a work in progress take the sliced audio/bpm/fps and hold an image for the duration.
|
||||
There is also a VHS converter node that allows you to load audio into the VHS video combine for audio insertion on the fly!
|
||||
|
||||
|
||||
|
||||
Directory Crawler: - Simple node that loads all images in a directory and any subdirectories as well
|
||||
-
|
||||
|
||||
|
||||
|
||||
|
||||
Raw Code Node: - Simple node that loads Python and allows you to dev inside comfy without having to reload the instance every time
|
||||
-
|
||||
Great for developing ideas and writing custom stuff quickly
|
||||
|
||||
|
||||
|
||||
|
||||
Glitch: Video and image effect
|
||||
-
|
||||
Slices up your image or video to make a glitching feel
|
||||
|
||||
|
||||
|
||||
Ripple: Video and image effect
|
||||
-
|
||||
Ripples your video or image
|
||||
|
||||
|
||||
|
||||
Pixel Sort: Video and image
|
||||
-
|
||||
CAUTION: This node is a very heavy operation. It takes 5-10 seconds per frame. (WIP)
|
||||
|
||||
|
||||
|
||||
Hexagon: Video an image
|
||||
-
|
||||
This one is really fun. It masks your image and video in slices, but thats not all! Each slice acts as its own video or image when you start rotating and messing with the parameters.
|
||||
|
||||
|
||||
|
||||
Ascii: Video and image
|
||||
-
|
||||
This one allows for a TON of different styles. This node also works with Alt Codes like this: alt+3 = ♥ or alt+219 = █
|
||||
If you play with the spacing of 219 you can actually get a pixel art effect. ALSO, the last character in the list will always be applied to the highest luminance areas of the image. This is useful because you can do silly things like leave the last character as a blank space, allowing for negative space to be applied to light areas.
|
||||
|
||||
<img width="1136" alt="Screenshot 2024-04-29 192646" src="https://github.com/filliptm/ComfyUI_Fill-Nodes/assets/55672949/926287e9-e22a-4e64-9e4f-7fd6e096b558">
|
||||
|
||||
Prompt Selector: - simple prompt selector/randomizer
|
||||
-
|
||||
Great for iterating through a lot of prompts or randomizing a list of things you already know works.
|
||||
|
||||
<img width="1021" alt="Screenshot 2024-05-15 144223" src="https://github.com/filliptm/ComfyUI_Fill-Nodes/assets/55672949/1e96e75a-e9f4-4056-a55a-0ee1cc316dff">
|
||||
|
||||
Random Number Range: - Randomize numbers within a selected range.
|
||||
-
|
||||
Has a lot of uses where you want numbers randomized, but you need them in a specific range for error purposes.
|
||||
|
||||
<img width="683" alt="Screenshot 2024-05-15 144242" src="https://github.com/filliptm/ComfyUI_Fill-Nodes/assets/55672949/b2d15b5c-aeda-44d1-b341-075940d83a3f">
|
||||
|
||||
Half Tone FX: - Creates a black and white half-tone effect.
|
||||
-
|
||||
Lots of fun with this one. Get interesting effects on both images and video.
|
||||
|
||||
<img width="953" alt="Screenshot 2024-05-15 144011" src="https://github.com/filliptm/ComfyUI_Fill-Nodes/assets/55672949/486e0e4f-ad37-4353-8e92-42e85767f882">
|
||||
|
||||
Infinite Zoom: - Creates a zooming effect for both images and video.
|
||||
-
|
||||
|
||||
|
||||
Paper Drawn: - Filter effect that makes your images and videos look like pencil drawn.
|
||||
-
|
||||
|
||||
|
||||
|
||||
+100
-783
@@ -1,783 +1,100 @@
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("fl_fill_nodes")
|
||||
|
||||
from . import routes
|
||||
|
||||
from .nodes.conditioning.FL_KreaReference import FL_KreaReference, FL_KreaReferenceGuider
|
||||
from .nodes.ksamplers.FL_KsamplerSEG_Krea import FL_KsamplerSEG_Krea
|
||||
|
||||
# AI NODES
|
||||
from .nodes.ai.FL_Fal_Gemini_ImageEdit import FL_Fal_Gemini_ImageEdit
|
||||
from .nodes.ai.FL_Fal_GPTImage2_Edit import FL_Fal_GPTImage2_Edit
|
||||
from .nodes.ai.FL_Fal_Kling_AIAvatar import FL_Fal_Kling_AIAvatar
|
||||
from .nodes.ai.FL_Fal_Kontext import FL_Fal_Kontext
|
||||
from .nodes.ai.FL_Fal_Pixverse import FL_Fal_Pixverse
|
||||
from .nodes.ai.FL_Fal_Pixverse_LipSync import FL_Fal_Pixverse_LipSync
|
||||
from .nodes.ai.FL_Fal_Pixverse_Transition import FL_Fal_Pixverse_Transition
|
||||
from .nodes.ai.FL_Fal_Seedance2_ReferenceToVideo import FL_Fal_Seedance2_ReferenceToVideo
|
||||
from .nodes.ai.FL_Fal_Seedance_i2v import FL_Fal_Seedance_i2v
|
||||
from .nodes.ai.FL_Fal_Seedream_Edit import FL_Fal_Seedream_Edit
|
||||
from .nodes.ai.FL_Fal_SeedVR_Upscale import FL_Fal_SeedVR_Upscale
|
||||
from .nodes.ai.FL_Fal_Sora import FL_Fal_Sora
|
||||
from .nodes.ai.FL_GeminiVideoCaptioner import FL_GeminiVideoCaptioner
|
||||
from .nodes.ai.FL_Hedra_API import FL_Hedra_API
|
||||
from .nodes.ai.FL_PixVerseAPI import FL_PixVerseAPI
|
||||
from .nodes.ai.FL_RunwayAct2 import FL_RunwayAct2
|
||||
from .nodes.ai.FL_RunwayImageAPI import FL_RunwayImageAPI
|
||||
|
||||
# google-genai nodes — wrapped so SDK import errors don't kill the entire pack
|
||||
_GENAI_NODES_AVAILABLE = True
|
||||
try:
|
||||
from .nodes.ai.FL_GeminiImageEditor import FL_GeminiImageEditor
|
||||
from .nodes.ai.FL_GeminiImageGenADV import FL_GeminiImageGenADV
|
||||
from .nodes.ai.FL_GeminiTextAPI import FL_GeminiTextAPI
|
||||
from .nodes.ai.FL_VertexGemini25FlashImage import FL_VertexGemini25FlashImage
|
||||
from .nodes.ai.FL_VertexVeo3 import FL_Veo3VideoGen
|
||||
except Exception as e:
|
||||
_GENAI_NODES_AVAILABLE = False
|
||||
logger.warning("Skipping Google Gemini/Vertex nodes due to SDK/import incompatibility: %s", e)
|
||||
logger.info("Install a compatible google-genai version to enable these nodes.")
|
||||
|
||||
# API_TOOLS NODES
|
||||
from .nodes.api_tools.FL_API_Base64_ImageLoader import FL_API_Base64_ImageLoader
|
||||
from .nodes.api_tools.FL_API_ImageSaver import FL_API_ImageSaver
|
||||
|
||||
# AUDIO NODES
|
||||
from .nodes.audio.FL_Audio_BPM_Analyzer import FL_Audio_BPM_Analyzer
|
||||
from .nodes.audio.FL_Audio_Beat_Prompt_Schedule import FL_Audio_Beat_Prompt_Schedule
|
||||
from .nodes.audio.FL_Prompt_Reference_Library import FL_Prompt_Reference_Library
|
||||
from .nodes.audio.FL_Audio_Prompt_Envelope import FL_Audio_Beat_Prompt_Envelope, FL_Audio_Envelope_Prompt
|
||||
from .nodes.audio.FL_Audio_Beat_Visualizer import FL_Audio_Beat_Visualizer
|
||||
from .nodes.audio.FL_Audio_Crop import FL_Audio_Crop
|
||||
from .nodes.audio.FL_Audio_Drum_Detector import FL_Audio_Drum_Detector
|
||||
from .nodes.audio.FL_Audio_Envelope_Visualizer import FL_Audio_Envelope_Visualizer
|
||||
from .nodes.audio.FL_Audio_Music_Video_Sequencer import FL_Audio_Music_Video_Sequencer
|
||||
from .nodes.audio.FL_Audio_Reactive_Brightness import FL_Audio_Reactive_Brightness
|
||||
from .nodes.audio.FL_Audio_Reactive_Edge_Glow import FL_Audio_Reactive_Edge_Glow
|
||||
from .nodes.audio.FL_Audio_Reactive_Envelope import FL_Audio_Reactive_Envelope
|
||||
from .nodes.audio.FL_Audio_Reactive_Saturation import FL_Audio_Reactive_Saturation
|
||||
from .nodes.audio.FL_Audio_Reactive_Scale import FL_Audio_Reactive_Scale
|
||||
from .nodes.audio.FL_Audio_Reactive_Speed import FL_Audio_Reactive_Speed
|
||||
from .nodes.audio.FL_Audio_Segment_Extractor import FL_Audio_Segment_Extractor
|
||||
from .nodes.audio.FL_Audio_Separation import FL_Audio_Separation
|
||||
from .nodes.audio.FL_Audio_Shot_Iterator import FL_Audio_Shot_Iterator
|
||||
from .nodes.audio.FL_AudioFrameCalculator import FL_AudioFrameCalculator
|
||||
|
||||
# CAPTIONING NODES
|
||||
from .nodes.captioning.FL_CSVExtractor import FL_CSVExtractor
|
||||
from .nodes.captioning.FL_CaptionToCSV import FL_CaptionToCSV
|
||||
from .nodes.captioning.FL_Caption_Saver_V2 import FL_CaptionSaver_V2
|
||||
from .nodes.captioning.FL_ImageCaptionLayout import FL_ImageCaptionLayout
|
||||
from .nodes.captioning.FL_ImageCaptionLayoutPDF import FL_ImageCaptionLayoutPDF
|
||||
from .nodes.captioning.FL_Image_Caption_Saver import FL_ImageCaptionSaver
|
||||
from .nodes.captioning.FL_LoadCSV import FL_LoadCSV
|
||||
from .nodes.captioning.FL_MirrorAndAppendCaptions import FL_MirrorAndAppendCaptions
|
||||
from .nodes.captioning.FL_OllamaCaptioner import FL_OllamaCaptioner
|
||||
from .nodes.captioning.FL_SaveCSV import FL_SaveCSV
|
||||
from .nodes.captioning.FL_Video_Caption_Saver import FL_VideoCaptionSaver
|
||||
from .nodes.captioning.FL_WordFrequencyGraph import FL_WordFrequencyGraph
|
||||
|
||||
# DISCORD NODES
|
||||
from .nodes.discord.FL_DiscordWebhook import FL_SendToDiscordWebhook
|
||||
|
||||
# EXPERIMENTS NODES
|
||||
from .nodes.experiments.FL_BatchAligned import FL_BatchAlign
|
||||
from .nodes.experiments.FL_ColorPicker import FL_ColorPicker
|
||||
from .nodes.experiments.FL_GradGen import FL_GradGenerator
|
||||
from .nodes.experiments.FL_NFTGenerator import FL_NFTGenerator
|
||||
|
||||
# FILE_OPERATIONS NODES
|
||||
from .nodes.file_operations.FL_ZipDirectory import FL_ZipDirectory
|
||||
from .nodes.file_operations.FL_ZipSave import FL_ZipSave
|
||||
|
||||
# GAMES NODES
|
||||
from .nodes.games.FL_BulletHellGame import FL_BulletHellGame
|
||||
from .nodes.games.FL_TetrisGame import FL_TetrisGame
|
||||
|
||||
# GOOGLE_DRIVE NODES
|
||||
from .nodes.google_drive.FL_GoogleCloudStorage import FL_GoogleCloudStorage
|
||||
from .nodes.google_drive.FL_GoogleDriveDownloader import FL_GoogleDriveDownloader
|
||||
from .nodes.google_drive.FL_GoogleDriveImageDownloader import FL_GoogleDriveImageDownloader
|
||||
|
||||
# GPT NODES
|
||||
from .nodes.gpt.FL_Dalle3 import FL_Dalle3
|
||||
from .nodes.gpt.FL_GPT_Image1 import FL_GPT_Image1
|
||||
from .nodes.gpt.FL_GPT_Image1_ADV import FL_GPT_Image1_ADV
|
||||
from .nodes.gpt.FL_GPT_Text import FL_GPT_Text
|
||||
from .nodes.gpt.FL_GPT_Vision import FL_GPT_Vision
|
||||
from .nodes.gpt.FL_SimpleGPTVision import FL_SimpleGPTVision
|
||||
|
||||
# HUGGING_FACE NODES
|
||||
from .nodes.hugging_face.FL_HFDatasetDownloader import FL_HFDatasetDownloader
|
||||
from .nodes.hugging_face.FL_HFHubModelUploader import FL_HFHubModelUploader
|
||||
from .nodes.hugging_face.FL_HF_Character import FL_HF_Character
|
||||
from .nodes.hugging_face.FL_HF_UploaderAbsolute import FL_HF_UploaderAbsolute
|
||||
|
||||
# KARTEL NODES
|
||||
from .nodes.kartel.FL_KartelJobInput import FL_KartelJobInput
|
||||
from .nodes.kartel.FL_KartelJobOutput import FL_KartelJobOutput
|
||||
|
||||
# IMAGE NODES
|
||||
from .nodes.image.FL_AnimeLineExtractor import FL_AnimeLineExtractor
|
||||
from .nodes.image.FL_ApplyMask import FL_ApplyMask
|
||||
from .nodes.image.FL_BlackFrameReject import FL_BlackFrameReject
|
||||
from .nodes.image.FL_CoTracker import FL_CoTracker
|
||||
from .nodes.image.FL_CreateShapeImageOnPath import FL_CreateShapeImageOnPath
|
||||
from .nodes.image.FL_ImageAddNoise import FL_ImageAddNoise
|
||||
from .nodes.image.FL_ImageAdjuster import FL_ImageAdjuster
|
||||
from .nodes.image.FL_ImageAspectCropper import FL_ImageAspectCropper
|
||||
from .nodes.image.FL_ImageBatch import FL_ImageBatch
|
||||
from .nodes.image.FL_ImageBatchListConverter import FL_ImageListToImageBatch, FL_ImageBatchToImageList
|
||||
from .nodes.image.FL_ImageBatchToGrid import FL_ImageBatchToGrid
|
||||
from .nodes.image.FL_ImageNotes import FL_ImageNotes
|
||||
from .nodes.image.FL_ImageOverlay import FL_ImageOverlay
|
||||
from .nodes.image.FL_ImageReplace import FL_ImageReplace
|
||||
from .nodes.image.FL_ImageSelector import FL_ImageSelector
|
||||
from .nodes.image.FL_ImagePicker import FL_ImagePicker
|
||||
from .nodes.image.FL_RegionPNGOverlay import FL_RegionPNGOverlay
|
||||
from .nodes.image.FL_ImageSlicer import FL_ImageSlicer
|
||||
from .nodes.image.FL_ImageToMask import FL_ImageToMask
|
||||
from .nodes.image.FL_Image_AddToBatch import FL_ImageAddToBatch
|
||||
from .nodes.image.FL_Image_Blank import FL_ImageBlank
|
||||
from .nodes.image.FL_Image_Crop import FL_ImageCrop
|
||||
from .nodes.image.FL_Image_Dimension_Display import FL_ImageDimensionDisplay
|
||||
from .nodes.image.FL_Image_Randomizer import FL_ImageRandomizer
|
||||
from .nodes.image.FL_LoadImage import FL_LoadImage
|
||||
from .nodes.image.FL_PaddingRemover import FL_PaddingRemover
|
||||
from .nodes.image.FL_ReplaceColor import FL_ReplaceColor
|
||||
from .nodes.image.FL_SaveAndDisplayImage import FL_SaveAndDisplayImage
|
||||
from .nodes.image.FL_SaveImages import FL_SaveImages
|
||||
from .nodes.image.FL_SaveRGBAAnimatedWebP import FL_SaveRGBAAnimatedWebP
|
||||
from .nodes.image.FL_SaveWebM import FL_SaveWebM
|
||||
from .nodes.image.FL_SaveWebpImages import FL_SaveWebPImage
|
||||
|
||||
# KSAMPLERS NODES
|
||||
from .nodes.ksamplers.FL_KsamplerBasic import FL_KsamplerBasic
|
||||
from .nodes.ksamplers.FL_KsamplerContextWindow import FL_KsamplerContextWindow
|
||||
from .nodes.ksamplers.FL_KsamplerPlus import FL_KsamplerPlus
|
||||
from .nodes.ksamplers.FL_KsamplerPlusV2 import FL_KsamplerPlusV2
|
||||
from .nodes.ksamplers.FL_KsamplerSigma import FL_KsamplerSigma
|
||||
from .nodes.ksamplers.FL_KsamplerSEG_Regions import FL_KsamplerSEG_Regions
|
||||
from .nodes.ksamplers.FL_KsamplerSEG_Captioner import FL_KsamplerSEG_Captioner
|
||||
from .nodes.ksamplers.FL_KsamplerSEG_Encoder import FL_KsamplerSEG_Encoder
|
||||
from .nodes.ksamplers.FL_KsamplerSEG import FL_KsamplerSEG, FL_KsamplerSEGAdvanced
|
||||
from .nodes.ksamplers.FL_KsamplerSettings import FL_KsamplerSettings
|
||||
from .nodes.ksamplers.FL_SamplerStrings import FL_SamplerStrings
|
||||
from .nodes.ksamplers.FL_SchedulerStrings import FL_SchedulerStrings
|
||||
from .nodes.ksamplers.FL_KSamplerXYZPlot import FL_KSamplerXYZPlot
|
||||
|
||||
# LOADERS NODES
|
||||
from .nodes.loaders.FL_NodeLoader import FL_NodeLoader
|
||||
from .nodes.loaders.FL_NodePackLoader import FL_NodePackLoader
|
||||
from .nodes.loaders.FL_UpscaleModel import FL_UpscaleModel
|
||||
|
||||
# PDF NODES
|
||||
from .nodes.pdf.FL_BulkPDFLoader import FL_BulkPDFLoader
|
||||
from .nodes.pdf.FL_ImagesToPDF import FL_ImagesToPDF
|
||||
from .nodes.pdf.FL_PDFEncryptor import FL_PDFEncryptor
|
||||
from .nodes.pdf.FL_PDFImageExtractor import FL_PDFImageExtractor
|
||||
from .nodes.pdf.FL_PDFLoader import FL_PDFLoader
|
||||
from .nodes.pdf.FL_PDFMerger import FL_PDFMerger
|
||||
from .nodes.pdf.FL_PDFSaver import FL_PDFSaver
|
||||
from .nodes.pdf.FL_PDFTextExtractor import FL_PDFTextExtractor
|
||||
from .nodes.pdf.FL_PDFToImage import FL_PDFToImages
|
||||
from .nodes.pdf.FL_TextToPDF import FL_TextToPDF
|
||||
|
||||
# PROMPTING NODES
|
||||
from .nodes.prompting.FL_MadLibGenerator import FL_MadLibGenerator
|
||||
from .nodes.prompting.FL_GenerateText import FL_GenerateText
|
||||
from .nodes.prompting.FL_Prompt import FL_PromptBasic
|
||||
from .nodes.prompting.FL_PromptMulti import FL_PromptMulti
|
||||
from .nodes.prompting.FL_PromptSelector import FL_PromptSelector
|
||||
from .nodes.prompting.FL_PromptSelectorBasic import FL_PromptSelectorBasic
|
||||
|
||||
# UTILITY NODES
|
||||
from .nodes.utility.FL_ClipScanner import FL_ClipScanner
|
||||
from .nodes.utility.FL_DirectoryCrawl import FL_DirectoryCrawl
|
||||
from .nodes.utility.FL_Float import FL_Float
|
||||
from .nodes.utility.FL_InpaintCrop import FL_InpaintCrop
|
||||
from .nodes.utility.FL_InpaintCrop import FL_Inpaint_Stitch
|
||||
from .nodes.utility.FL_JS import FL_JS
|
||||
from .nodes.utility.FL_Math import FL_Math
|
||||
from .nodes.utility.FL_ModelInspector import FL_ModelInspector
|
||||
from .nodes.utility.FL_NumberConverter import FL_IntToFloat, FL_FloatToInt
|
||||
from .nodes.utility.FL_Padding import FL_Padding
|
||||
from .nodes.utility.FL_PasteByMask import FL_PasteByMask
|
||||
from .nodes.utility.FL_PasteOnCanvas import FL_PasteOnCanvas
|
||||
from .nodes.utility.FL_PathTypeChecker import FL_PathTypeChecker
|
||||
from .nodes.utility.FL_RandomRange import FL_RandomNumber
|
||||
from .nodes.utility.FL_RandomShapeGenerator import FL_RandomShapeGenerator
|
||||
from .nodes.utility.FL_SD_Slices import FL_SDUltimate_Slices
|
||||
from .nodes.utility.FL_SeparateMasks import FL_SeparateMaskComponents
|
||||
from .nodes.utility.FL_ShowText import FL_ShowText
|
||||
from .nodes.utility.FL_StringToLoraName import FL_StringToLoraName
|
||||
from .nodes.utility.FL_ModelDifferenceLoraSave import FL_ModelDifferenceLoraSave
|
||||
from .nodes.utility.FL_Switch import FL_Switch
|
||||
from .nodes.utility.FL_Switch_Big import FL_Switch_Big
|
||||
from .nodes.utility.FL_SystemCheck import FL_SystemCheck
|
||||
from .nodes.utility.FL_UnloadModel import FL_UnloadModel, FL_UnloadAllModels
|
||||
from .nodes.utility.FL_VideoCropNStitch import FL_VideoCropMask
|
||||
from .nodes.utility.FL_VideoCropNStitch import FL_VideoRecompose
|
||||
|
||||
# VFX NODES
|
||||
from .nodes.vfx.FL_Ascii import FL_Ascii
|
||||
from .nodes.vfx.FL_DepthBlur import FL_DepthBlur
|
||||
from .nodes.vfx.FL_Dither import FL_Dither
|
||||
from .nodes.vfx.FL_Glitch import FL_Glitch
|
||||
from .nodes.vfx.FL_StreetScan import FL_ScanVideoDetections, FL_StreetScanComposite
|
||||
from .nodes.vfx.FL_ScanAudioEdit import FL_ScanAudioEdit
|
||||
from .nodes.vfx.FL_VoxelNormalRelief import FL_VoxelNormalRelief
|
||||
from .nodes.vfx.FL_LayeredParallax import FL_ParallaxLayer, FL_LayeredParallax, FL_ParallaxDepthSources, FL_ParallaxStackFromBatch
|
||||
from .nodes.vfx.FL_PosterLayers import FL_PosterLayerPlanner, FL_PosterLayers, FL_PosterLayerAsset, FL_PosterLayerStack
|
||||
from .nodes.vfx.poster_layer_cache import PosterLayerCache
|
||||
from comfy_execution.cache_provider import register_cache_provider
|
||||
|
||||
from .nodes.vfx.FL_InteractiveScanFX import FL_InteractiveScanFX, FL_ScanAnalysis, FL_ScanVideoSection, FL_ScanVideoShots, FL_ScanAnalysisCollect
|
||||
from .nodes.vfx.FL_HalfTone import FL_HalftonePattern
|
||||
from .nodes.vfx.FL_HexagonalPattern import FL_HexagonalPattern
|
||||
from .nodes.vfx.FL_ImageCollage import FL_ImageCollage
|
||||
from .nodes.vfx.FL_Image_Pixelator import FL_ImagePixelator
|
||||
from .nodes.vfx.FL_InfiniteZoom import FL_InfiniteZoom
|
||||
from .nodes.vfx.FL_PaperDrawn import FL_PaperDrawn
|
||||
from .nodes.vfx.FL_PixelArt import FL_PixelArtShader
|
||||
from .nodes.vfx.FL_PixelSort import FL_PixelSort
|
||||
from .nodes.vfx.FL_RetroEffect import FL_RetroEffect
|
||||
from .nodes.vfx.FL_Ripple import FL_Ripple
|
||||
from .nodes.vfx.FL_Shader import FL_Shadertoy
|
||||
from .nodes.vfx.FL_TextOverlay import FL_TextOverlayNode
|
||||
|
||||
# VIDEO NODES
|
||||
from .nodes.video.FL_FILM import FL_FILM
|
||||
from .nodes.video.FL_ProResVideo import FL_ProResVideo
|
||||
from .nodes.video.FL_RIFE import FL_RIFE
|
||||
from .nodes.video.FL_VideoBatchSplitter import FL_VideoBatchSplitter
|
||||
from .nodes.video.FL_VideoCadence import FL_VideoCadence
|
||||
from .nodes.video.FL_VideoCadenceCompile import FL_VideoCadenceCompile
|
||||
from .nodes.video.FL_VideoCrossfade import FL_VideoCrossfade
|
||||
from .nodes.video.FL_VideoCut import FL_VideoCut
|
||||
from .nodes.video.FL_LoadVideo import FL_LoadVideo
|
||||
from .nodes.video.FL_VideoCombine import FL_VideoCombine
|
||||
from .nodes.video.FL_VideoPicker import FL_VideoPicker
|
||||
from .nodes.video.FL_VideoTrim import FL_VideoTrim
|
||||
|
||||
# WIP NODES
|
||||
from .nodes.wip.FL_KsamplerFractals import FL_FractalKSampler
|
||||
from .nodes.wip.FL_TimeLine import FL_TimeLine
|
||||
from .nodes.wip.FL_WF_Agent import FL_WF_Agent
|
||||
from .nodes.wip.FL_WanFirstLastFrameToVideo import FL_WanFirstLastFrameToVideo
|
||||
from .nodes.wip.FL_QwenImageEditStrength import FL_QwenImageEditStrength
|
||||
from .nodes.wip.FL_WanVaceToVideoMultiRef import FL_WanVaceToVideoMultiRef
|
||||
from .nodes.wip.FL_AnimatedShapePatterns import FL_AnimatedShapePatterns
|
||||
from .nodes.wip.FL_PathAnimator import FL_PathAnimator
|
||||
from .nodes.wip.FL_WanVideoContinue import FL_WanVideoContinue
|
||||
from .nodes.wip.FL_WanVideoBlender import FL_WanVideoBlender
|
||||
from .nodes.wip.FL_WanVideoBlender3Way import FL_WanVideoBlender3Way
|
||||
from .nodes.wip.FL_WanVideoContinuationBlender import FL_WanVideoContinuationBlender
|
||||
from .nodes.wip.FL_ZImageControlNetPatch import FL_ZImageControlNetPatch
|
||||
from .nodes.wip.FL_LTXVMaskedICLoRAGuide import FL_LTXVMaskedICLoRAGuide
|
||||
|
||||
register_cache_provider(PosterLayerCache())
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FL_KreaReference": FL_KreaReference,
|
||||
"FL_KreaReferenceGuider": FL_KreaReferenceGuider,
|
||||
"FL_KsamplerSEG_Krea": FL_KsamplerSEG_Krea,
|
||||
"FL_SaveWebM": FL_SaveWebM,
|
||||
"FL_TextOverlayNode": FL_TextOverlayNode,
|
||||
"FL_ImageBlank": FL_ImageBlank,
|
||||
"FL_ImageRandomizer": FL_ImageRandomizer,
|
||||
"FL_ImageCaptionSaver": FL_ImageCaptionSaver,
|
||||
"FL_VideoCaptionSaver": FL_VideoCaptionSaver,
|
||||
"FL_ImageDimensionDisplay": FL_ImageDimensionDisplay,
|
||||
"FL_GeminiVideoCaptioner": FL_GeminiVideoCaptioner,
|
||||
"FL_GPT_Image1": FL_GPT_Image1,
|
||||
"FL_ImagePixelator": FL_ImagePixelator,
|
||||
"FL_ImageAddToBatch": FL_ImageAddToBatch,
|
||||
"FL_DirectoryCrawl": FL_DirectoryCrawl,
|
||||
"FL_Ascii": FL_Ascii,
|
||||
"FL_ReplaceColor": FL_ReplaceColor,
|
||||
"FL_ImageAddNoise": FL_ImageAddNoise,
|
||||
"FL_WordFrequencyGraph": FL_WordFrequencyGraph,
|
||||
"FL_Glitch": FL_Glitch,
|
||||
"FL_ScanVideoDetections": FL_ScanVideoDetections,
|
||||
"FL_StreetScanComposite": FL_StreetScanComposite,
|
||||
"FL_ScanAudioEdit": FL_ScanAudioEdit,
|
||||
"FL_VoxelNormalRelief": FL_VoxelNormalRelief,
|
||||
"FL_ParallaxLayer": FL_ParallaxLayer,
|
||||
"FL_LayeredParallax": FL_LayeredParallax,
|
||||
"FL_ParallaxDepthSources": FL_ParallaxDepthSources,
|
||||
"FL_ParallaxStackFromBatch": FL_ParallaxStackFromBatch,
|
||||
"FL_PosterLayerPlanner": FL_PosterLayerPlanner,
|
||||
"FL_PosterLayers": FL_PosterLayers,
|
||||
"FL_PosterLayerAsset": FL_PosterLayerAsset,
|
||||
"FL_PosterLayerStack": FL_PosterLayerStack,
|
||||
"FL_InteractiveScanFX": FL_InteractiveScanFX,
|
||||
"FL_ScanAnalysis": FL_ScanAnalysis,
|
||||
"FL_ScanVideoSection": FL_ScanVideoSection,
|
||||
"FL_ScanVideoShots": FL_ScanVideoShots,
|
||||
"FL_ScanAnalysisCollect": FL_ScanAnalysisCollect,
|
||||
"FL_Ripple": FL_Ripple,
|
||||
"FL_PixelSort": FL_PixelSort,
|
||||
"FL_HexagonalPattern": FL_HexagonalPattern,
|
||||
"FL_NFTGenerator": FL_NFTGenerator,
|
||||
"FL_HalftonePattern": FL_HalftonePattern,
|
||||
"FL_DepthBlur": FL_DepthBlur,
|
||||
"FL_RandomNumber": FL_RandomNumber,
|
||||
"FL_PromptSelector": FL_PromptSelector,
|
||||
"FL_GenerateText": FL_GenerateText,
|
||||
"FL_Shadertoy": FL_Shadertoy,
|
||||
"FL_PixelArtShader": FL_PixelArtShader,
|
||||
"FL_InfiniteZoom": FL_InfiniteZoom,
|
||||
"FL_PaperDrawn": FL_PaperDrawn,
|
||||
"FL_ImageNotes": FL_ImageNotes,
|
||||
"FL_ImageCollage": FL_ImageCollage,
|
||||
"FL_KsamplerSettings": FL_KsamplerSettings,
|
||||
"FL_RetroEffect": FL_RetroEffect,
|
||||
"FL_InpaintCrop": FL_InpaintCrop,
|
||||
"FL_Inpaint_Stitch": FL_Inpaint_Stitch,
|
||||
"FL_SDUltimate_Slices": FL_SDUltimate_Slices,
|
||||
"FL_BatchAlign": FL_BatchAlign,
|
||||
"FL_VideoRecompose": FL_VideoRecompose,
|
||||
"FL_VideoCropMask": FL_VideoCropMask,
|
||||
"FL_SeparateMaskComponents": FL_SeparateMaskComponents,
|
||||
"FL_PasteOnCanvas": FL_PasteOnCanvas,
|
||||
"FL_BulletHellGame": FL_BulletHellGame,
|
||||
"FL_TetrisGame": FL_TetrisGame,
|
||||
"FL_Dither": FL_Dither,
|
||||
"FL_SystemCheck": FL_SystemCheck,
|
||||
"FL_ColorPicker": FL_ColorPicker,
|
||||
"FL_GradGenerator": FL_GradGenerator,
|
||||
"FL_MirrorAndAppendCaptions": FL_MirrorAndAppendCaptions,
|
||||
"FL_ImageCaptionLayout": FL_ImageCaptionLayout,
|
||||
"FL_HFHubModelUploader": FL_HFHubModelUploader,
|
||||
"FL_ZipDirectory": FL_ZipDirectory,
|
||||
"FL_ZipSave": FL_ZipSave,
|
||||
"FL_GPT_Vision": FL_GPT_Vision,
|
||||
"FL_TimeLine": FL_TimeLine,
|
||||
"FL_SimpleGPTVision": FL_SimpleGPTVision,
|
||||
"FL_SendToDiscordWebhook": FL_SendToDiscordWebhook,
|
||||
"FL_HF_Character": FL_HF_Character,
|
||||
"FL_CaptionToCSV": FL_CaptionToCSV,
|
||||
"FL_KsamplerPlus": FL_KsamplerPlus,
|
||||
"FL_KsamplerPlusV2": FL_KsamplerPlusV2,
|
||||
"FL_KsamplerBasic": FL_KsamplerBasic,
|
||||
"FL_KsamplerContextWindow": FL_KsamplerContextWindow,
|
||||
"FL_KsamplerSigma": FL_KsamplerSigma,
|
||||
"FL_KsamplerSEG_Regions": FL_KsamplerSEG_Regions,
|
||||
"FL_KsamplerSEG_Captioner": FL_KsamplerSEG_Captioner,
|
||||
"FL_KsamplerSEG_Encoder": FL_KsamplerSEG_Encoder,
|
||||
"FL_KsamplerSEG": FL_KsamplerSEG,
|
||||
"FL_KsamplerSEGAdvanced": FL_KsamplerSEGAdvanced,
|
||||
"FL_FractalKSampler": FL_FractalKSampler,
|
||||
"FL_UpscaleModel": FL_UpscaleModel,
|
||||
"FL_SaveCSV": FL_SaveCSV,
|
||||
"FL_LoadCSV": FL_LoadCSV,
|
||||
"FL_CSVExtractor": FL_CSVExtractor,
|
||||
"FL_KSamplerXYZPlot": FL_KSamplerXYZPlot,
|
||||
"FL_SamplerStrings": FL_SamplerStrings,
|
||||
"FL_SchedulerStrings": FL_SchedulerStrings,
|
||||
"FL_ImageCaptionLayoutPDF": FL_ImageCaptionLayoutPDF,
|
||||
"FL_Dalle3": FL_Dalle3,
|
||||
"FL_SaveImages": FL_SaveImages,
|
||||
"FL_LoadImage": FL_LoadImage,
|
||||
"FL_PDFLoader": FL_PDFLoader,
|
||||
"FL_PDFToImages": FL_PDFToImages,
|
||||
"FL_PDFSaver": FL_PDFSaver,
|
||||
"FL_ImagesToPDF": FL_ImagesToPDF,
|
||||
"FL_PDFMerger": FL_PDFMerger,
|
||||
"FL_PDFTextExtractor": FL_PDFTextExtractor,
|
||||
"FL_PDFImageExtractor": FL_PDFImageExtractor,
|
||||
"FL_BulkPDFLoader": FL_BulkPDFLoader,
|
||||
"FL_TextToPDF": FL_TextToPDF,
|
||||
"FL_PDFEncryptor": FL_PDFEncryptor,
|
||||
"FL_SaveAndDisplayImage": FL_SaveAndDisplayImage,
|
||||
"FL_OllamaCaptioner": FL_OllamaCaptioner,
|
||||
"FL_ImageAdjuster": FL_ImageAdjuster,
|
||||
"FL_CaptionSaver_V2": FL_CaptionSaver_V2,
|
||||
"FL_PathTypeChecker": FL_PathTypeChecker,
|
||||
"FL_SaveWebPImage(SaveImage)": FL_SaveWebPImage,
|
||||
"FL_MadLibGenerator": FL_MadLibGenerator,
|
||||
"FL_Math": FL_Math,
|
||||
"FL_ImageSlicer": FL_ImageSlicer,
|
||||
"FL_ImageSelector": FL_ImageSelector,
|
||||
"FL_ImagePicker": FL_ImagePicker,
|
||||
"FL_ImageOverlay": FL_ImageOverlay,
|
||||
"FL_RegionPNGOverlay": FL_RegionPNGOverlay,
|
||||
"FL_ImageReplace": FL_ImageReplace,
|
||||
"FL_ImageAspectCropper": FL_ImageAspectCropper,
|
||||
"FL_CoTracker": FL_CoTracker,
|
||||
"FL_CreateShapeImageOnPath": FL_CreateShapeImageOnPath,
|
||||
"FL_HF_UploaderAbsolute": FL_HF_UploaderAbsolute,
|
||||
"FL_ImageListToImageBatch": FL_ImageListToImageBatch,
|
||||
"FL_ImageBatchToImageList": FL_ImageBatchToImageList,
|
||||
"FL_ImageBatchToGrid": FL_ImageBatchToGrid,
|
||||
"FL_ApplyMask": FL_ApplyMask,
|
||||
"FL_ProResVideo": FL_ProResVideo,
|
||||
"FL_LoadVideo": FL_LoadVideo,
|
||||
"FL_VideoCombine": FL_VideoCombine,
|
||||
"FL_VideoPicker": FL_VideoPicker,
|
||||
"FL_Padding": FL_Padding,
|
||||
"FL_GoogleDriveDownloader": FL_GoogleDriveDownloader,
|
||||
"FL_NodeLoader": FL_NodeLoader,
|
||||
"FL_NodePackLoader": FL_NodePackLoader,
|
||||
"FL_API_Base64_ImageLoader": FL_API_Base64_ImageLoader,
|
||||
"FL_API_ImageSaver": FL_API_ImageSaver,
|
||||
"FL_GoogleDriveImageDownloader": FL_GoogleDriveImageDownloader,
|
||||
"FL_AnimeLineExtractor": FL_AnimeLineExtractor,
|
||||
"FL_ClipScanner": FL_ClipScanner,
|
||||
"FL_VideoCut": FL_VideoCut,
|
||||
"FL_JS": FL_JS,
|
||||
"FL_HFDatasetDownloader": FL_HFDatasetDownloader,
|
||||
"FL_WF_Agent": FL_WF_Agent,
|
||||
"FL_BlackFrameReject": FL_BlackFrameReject,
|
||||
"FL_PixVerseAPI": FL_PixVerseAPI,
|
||||
"FL_Fal_Pixverse": FL_Fal_Pixverse,
|
||||
"FL_Fal_Kontext": FL_Fal_Kontext,
|
||||
"FL_Fal_GPTImage2_Edit": FL_Fal_GPTImage2_Edit,
|
||||
"FL_Fal_Gemini_ImageEdit": FL_Fal_Gemini_ImageEdit,
|
||||
"FL_Fal_Seedance2_ReferenceToVideo": FL_Fal_Seedance2_ReferenceToVideo,
|
||||
"FL_Fal_Seedance_i2v": FL_Fal_Seedance_i2v,
|
||||
"FL_Fal_Seedream_Edit": FL_Fal_Seedream_Edit,
|
||||
"FL_Fal_SeedVR_Upscale": FL_Fal_SeedVR_Upscale,
|
||||
"FL_Fal_Pixverse_Transition": FL_Fal_Pixverse_Transition,
|
||||
"FL_Fal_Pixverse_LipSync": FL_Fal_Pixverse_LipSync,
|
||||
"FL_Fal_Kling_AIAvatar": FL_Fal_Kling_AIAvatar,
|
||||
"FL_Fal_Sora": FL_Fal_Sora,
|
||||
"FL_PromptBasic": FL_PromptBasic,
|
||||
"FL_PromptMulti": FL_PromptMulti,
|
||||
"FL_PromptSelectorBasic": FL_PromptSelectorBasic,
|
||||
"FL_PaddingRemover": FL_PaddingRemover,
|
||||
"FL_GPT_Text": FL_GPT_Text,
|
||||
"FL_GoogleCloudStorage": FL_GoogleCloudStorage,
|
||||
"FL_StringToLoraName": FL_StringToLoraName,
|
||||
"FL_ModelDifferenceLoraSave": FL_ModelDifferenceLoraSave,
|
||||
"FL_Switch": FL_Switch,
|
||||
"FL_Switch_Big": FL_Switch_Big,
|
||||
"FL_PasteByMask": FL_PasteByMask,
|
||||
"FL_ModelInspector": FL_ModelInspector,
|
||||
"FL_Float": FL_Float,
|
||||
"FL_UnloadModel": FL_UnloadModel,
|
||||
"FL_UnloadAllModels": FL_UnloadAllModels,
|
||||
"FL_VideoTrim": FL_VideoTrim,
|
||||
"FL_VideoBatchSplitter": FL_VideoBatchSplitter,
|
||||
"FL_VideoCrossfade": FL_VideoCrossfade,
|
||||
"FL_VideoCadence": FL_VideoCadence,
|
||||
"FL_VideoCadenceCompile": FL_VideoCadenceCompile,
|
||||
"FL_WanVaceToVideoMultiRef": FL_WanVaceToVideoMultiRef,
|
||||
"FL_RIFE": FL_RIFE,
|
||||
"FL_FILM": FL_FILM,
|
||||
"FL_GPT_Image1_ADV": FL_GPT_Image1_ADV,
|
||||
"FL_ImageBatch": FL_ImageBatch,
|
||||
"FL_Hedra_API": FL_Hedra_API,
|
||||
"FL_RunwayImageAPI": FL_RunwayImageAPI,
|
||||
"FL_RunwayAct2": FL_RunwayAct2,
|
||||
"FL_ImageCrop": FL_ImageCrop,
|
||||
"FL_ImageToMask": FL_ImageToMask,
|
||||
"FL_WanFirstLastFrameToVideo": FL_WanFirstLastFrameToVideo,
|
||||
"FL_WanVideoContinue": FL_WanVideoContinue,
|
||||
"FL_WanVideoBlender": FL_WanVideoBlender,
|
||||
"FL_WanVideoBlender3Way": FL_WanVideoBlender3Way,
|
||||
"FL_WanVideoContinuationBlender": FL_WanVideoContinuationBlender,
|
||||
"FL_ZImageControlNetPatch": FL_ZImageControlNetPatch,
|
||||
"FL_LTXVMaskedICLoRAGuide": FL_LTXVMaskedICLoRAGuide,
|
||||
"FL_SaveRGBAAnimatedWebP": FL_SaveRGBAAnimatedWebP,
|
||||
"FL_Audio_BPM_Analyzer": FL_Audio_BPM_Analyzer,
|
||||
"FL_Audio_Beat_Prompt_Schedule": FL_Audio_Beat_Prompt_Schedule,
|
||||
"FL_Prompt_Reference_Library": FL_Prompt_Reference_Library,
|
||||
"FL_Audio_Beat_Prompt_Envelope": FL_Audio_Beat_Prompt_Envelope,
|
||||
"FL_Audio_Envelope_Prompt": FL_Audio_Envelope_Prompt,
|
||||
"FL_Audio_Beat_Visualizer": FL_Audio_Beat_Visualizer,
|
||||
"FL_Audio_Crop": FL_Audio_Crop,
|
||||
"FL_Audio_Drum_Detector": FL_Audio_Drum_Detector,
|
||||
"FL_Audio_Envelope_Visualizer": FL_Audio_Envelope_Visualizer,
|
||||
"FL_Audio_Music_Video_Sequencer": FL_Audio_Music_Video_Sequencer,
|
||||
"FL_Audio_Reactive_Brightness": FL_Audio_Reactive_Brightness,
|
||||
"FL_Audio_Reactive_Edge_Glow": FL_Audio_Reactive_Edge_Glow,
|
||||
"FL_Audio_Reactive_Envelope": FL_Audio_Reactive_Envelope,
|
||||
"FL_Audio_Reactive_Saturation": FL_Audio_Reactive_Saturation,
|
||||
"FL_Audio_Reactive_Scale": FL_Audio_Reactive_Scale,
|
||||
"FL_Audio_Reactive_Speed": FL_Audio_Reactive_Speed,
|
||||
"FL_Audio_Segment_Extractor": FL_Audio_Segment_Extractor,
|
||||
"FL_Audio_Separation": FL_Audio_Separation,
|
||||
"FL_Audio_Shot_Iterator": FL_Audio_Shot_Iterator,
|
||||
"FL_AudioFrameCalculator": FL_AudioFrameCalculator,
|
||||
"FL_QwenImageEditStrength": FL_QwenImageEditStrength,
|
||||
"FL_IntToFloat": FL_IntToFloat,
|
||||
"FL_FloatToInt": FL_FloatToInt,
|
||||
"FL_AnimatedShapePatterns": FL_AnimatedShapePatterns,
|
||||
"FL_PathAnimator": FL_PathAnimator,
|
||||
"FL_RandomShapeGenerator": FL_RandomShapeGenerator,
|
||||
"FL_KartelJobInput": FL_KartelJobInput,
|
||||
"FL_KartelJobOutput": FL_KartelJobOutput,
|
||||
"FL_ShowText": FL_ShowText,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_ImageBlank": "FL Image Blank",
|
||||
"FL_ImageRandomizer": "FL Image Randomizer",
|
||||
"FL_ImageCaptionSaver": "FL Image Caption Saver",
|
||||
"FL_VideoCaptionSaver": "FL Video Caption Saver",
|
||||
"FL_ImageDimensionDisplay": "FL Image Size",
|
||||
"FL_GeminiVideoCaptioner": "FL Gemini Video Captioner",
|
||||
"FL_GPT_Image1": "FL GPT Image-1",
|
||||
"FL_ImagePixelator": "FL Image Pixelator",
|
||||
"FL_ImageAddToBatch": "FL Image Add To Batch",
|
||||
"FL_DirectoryCrawl": "FL Directory Crawl",
|
||||
"FL_Ascii": "FL Ascii",
|
||||
"FL_ReplaceColor": "FL Replace Color",
|
||||
"FL_ImageAddNoise": "FL Image Add Noise",
|
||||
"FL_WordFrequencyGraph": "FL Word Frequency Graph",
|
||||
"FL_Glitch": "FL Glitch",
|
||||
"FL_ScanVideoDetections": "FL Scan Video Detections",
|
||||
"FL_StreetScanComposite": "FL Street Scan Composite",
|
||||
"FL_ScanAudioEdit": "FL Scan Audio Edit",
|
||||
"FL_VoxelNormalRelief": "FL Voxel Normal Relief",
|
||||
"FL_ParallaxLayer": "FL Parallax Layer",
|
||||
"FL_LayeredParallax": "FL Layered Parallax",
|
||||
"FL_InteractiveScanFX": "FL Interactive Scan FX",
|
||||
"FL_ScanAnalysis": "FL Scan Analysis",
|
||||
"FL_ScanVideoSection": "FL Scan Video Section",
|
||||
"FL_Ripple": "FL Ripple",
|
||||
"FL_PixelSort": "FL PixelSort",
|
||||
"FL_HexagonalPattern": "FL Hexagonal Pattern",
|
||||
"FL_NFTGenerator": "FL NFT Generator",
|
||||
"FL_HalftonePattern": "FL Halftone",
|
||||
"FL_DepthBlur": "FL Depth Blur",
|
||||
"FL_RandomNumber": "FL Random Number",
|
||||
"FL_PromptSelector": "FL Prompt Selector",
|
||||
"FL_GenerateText": "FL Generate Text",
|
||||
"FL_PromptSelectorBasic": "FL Prompt Selector Basic",
|
||||
"FL_Shadertoy": "FL Shadertoy",
|
||||
"FL_PixelArtShader": "FL Pixel Art",
|
||||
"FL_InfiniteZoom": "FL Infinite Zoom",
|
||||
"FL_PaperDrawn": "FL Paper Drawn",
|
||||
"FL_ImageNotes": "FL Image Notes",
|
||||
"FL_ImageCollage": "FL Image Collage",
|
||||
"FL_KsamplerSettings": "FL KSampler Settings",
|
||||
"FL_RetroEffect": "FL Retro Effect",
|
||||
"FL_InpaintCrop": "FL Inpaint Crop",
|
||||
"FL_Inpaint_Stitch": "FL Inpaint Stitch",
|
||||
"FL_SDUltimate_Slices": "FL SDUltimate Slices",
|
||||
"FL_BatchAlign": "FL Batch Align",
|
||||
"FL_VideoCropMask": "FL Video CropMask",
|
||||
"FL_VideoRecompose": "FL Video Recompose",
|
||||
"FL_SeparateMaskComponents": "FL Separate Mask Components",
|
||||
"FL_PasteOnCanvas": "FL Paste On Canvas",
|
||||
"FL_BulletHellGame": "FL BulletHell Game",
|
||||
"FL_TetrisGame": "FL Tetris Game",
|
||||
"FL_Dither": "FL Dither",
|
||||
"FL_SystemCheck": "FL System Check",
|
||||
"FL_ColorPicker": "FL Color Picker",
|
||||
"FL_GradGenerator": "FL Grad Generator",
|
||||
"FL_MirrorAndAppendCaptions": "FL Mirror And Append Captions",
|
||||
"FL_ImageCaptionLayout": "FL Image Caption Layout",
|
||||
"FL_HFHubModelUploader": "FL HFHub Model Uploader",
|
||||
"FL_ZipDirectory": "FL Zip Directory",
|
||||
"FL_ZipSave": "FL_ZipSave",
|
||||
"FL_GPT_Vision": "FL GPT Captions",
|
||||
"FL_TimeLine": "FL Time Line",
|
||||
"FL_SimpleGPTVision": "FL Simple GPT Vision",
|
||||
"FL_SendToDiscordWebhook": "FL Kytra Discord Webhook",
|
||||
"FL_HF_Character": "FL HF Character",
|
||||
"FL_CaptionToCSV": "FL Caption To CSV",
|
||||
"FL_KsamplerPlus": "FL KSampler Plus",
|
||||
"FL_KsamplerPlusV2": "FL KSampler Plus V2",
|
||||
"FL_KsamplerBasic": "FL KSampler Basic",
|
||||
"FL_KsamplerContextWindow": "FL Context Window KSampler",
|
||||
"FL_KsamplerSigma": "FL KSampler Sigma",
|
||||
"FL_KsamplerSEG_Regions": "FL KSampler SEG Regions",
|
||||
"FL_KsamplerSEG_Captioner": "FL KSampler SEG Captioner",
|
||||
"FL_KsamplerSEG_Encoder": "FL KSampler SEG Encoder",
|
||||
"FL_KsamplerSEG": "FL KSampler SEG",
|
||||
"FL_KsamplerSEGAdvanced": "FL KSampler SEG Advanced",
|
||||
"FL_KsamplerSEG_Krea": "FL KSampler SEG Krea",
|
||||
"FL_FractalKSampler": "FL Fractal KSampler",
|
||||
"FL_UpscaleModel": "FL Upscale Model",
|
||||
"FL_SaveCSV": "FL Save CSV",
|
||||
"FL_LoadCSV": "FL Load CSV",
|
||||
"FL_CSVExtractor": "FL CSV Extractor",
|
||||
"FL_KSamplerXYZPlot": "FL KSampler XYZ Plot",
|
||||
"FL_SamplerStrings": "FL Sampler String XYZ",
|
||||
"FL_SchedulerStrings": "FL Scheduler String XYZ",
|
||||
"FL_ImageCaptionLayoutPDF": "FL Image Caption Layout PDF",
|
||||
"FL_Dalle3": "FL Dalle 3",
|
||||
"FL_SaveImages": "FL Save Images",
|
||||
"FL_LoadImage": "FL Load Image",
|
||||
"FL_PDFLoader": "FL PDF Loader",
|
||||
"FL_PDFToImages": "FL PDF To Images",
|
||||
"FL_PDFSaver": "FL PDF Saver",
|
||||
"FL_ImagesToPDF": "FL Images To PDF",
|
||||
"FL_PDFMerger": "FL PDF Merger",
|
||||
"FL_PDFTextExtractor": "FL PDF Text Extractor",
|
||||
"FL_PDFImageExtractor": "FL PDF Image Extractor",
|
||||
"FL_BulkPDFLoader": "FL Bulk PDF Loader",
|
||||
"FL_TextToPDF": "FL Text To PDF",
|
||||
"FL_PDFEncryptor": "FL PDF Encryptor",
|
||||
"FL_SaveAndDisplayImage": "FL Save And Display Image",
|
||||
"FL_OllamaCaptioner": "FL Ollama Captioner by Cosmic",
|
||||
"FL_ImageAdjuster": "FL Image Adjuster",
|
||||
"FL_CaptionSaver_V2": "FL Caption Saver V2",
|
||||
"FL_PathTypeChecker": "FL Path Type Checker",
|
||||
"FL_SaveWebPImage(SaveImage)": "FL Save WebP Image",
|
||||
"FL_MadLibGenerator": "FL MadLib Generator",
|
||||
"FL_Math": "FL Math",
|
||||
"FL_ImageSlicer": "FL Image Slicer",
|
||||
"FL_ImageSelector": "FL Image Selector",
|
||||
"FL_ImagePicker": "FL Image Picker",
|
||||
"FL_ImageOverlay": "FL Image Overlay",
|
||||
"FL_RegionPNGOverlay": "FL Region PNG Overlay",
|
||||
"FL_ImageReplace": "FL Image Replace",
|
||||
"FL_ImageAspectCropper": "FL Image Aspect Cropper",
|
||||
"FL_CoTracker": "FL CoTracker",
|
||||
"FL_CreateShapeImageOnPath": "FL Create Shape Image On Path",
|
||||
"FL_HF_UploaderAbsolute": "FL HF Uploader Absolute",
|
||||
"FL_ImageListToImageBatch": "FL Image List To Image Batch",
|
||||
"FL_ImageBatchToImageList": "FL Image Batch To Image List",
|
||||
"FL_ImageBatchToGrid": "FL Image Batch To Grid",
|
||||
"FL_ApplyMask": "FL Apply Mask",
|
||||
"FL_ProResVideo": "FL ProRes Video",
|
||||
"FL_LoadVideo": "FL Load Video",
|
||||
"FL_VideoCombine": "FL Video Combine",
|
||||
"FL_VideoPicker": "FL Video Picker",
|
||||
"FL_Padding": "FL Padding",
|
||||
"FL_GoogleDriveDownloader": "FL Google Drive Downloader",
|
||||
"FL_NodeLoader": "FL Node Loader",
|
||||
"FL_NodePackLoader": "FL Node Pack Loader",
|
||||
"FL_API_Base64_ImageLoader": "FL API Base64 Image Loader",
|
||||
"FL_API_ImageSaver": "FL API Image Saver",
|
||||
"FL_GoogleDriveImageDownloader": "FL Google Drive Image Downloader",
|
||||
"FL_AnimeLineExtractor": "FL Anime Line Extractor",
|
||||
"FL_ClipScanner": "FL Clip Scanner (Kytra)",
|
||||
"FL_VideoCut": "FL Video Cut Detector",
|
||||
"FL_JS": "FL JavaScript",
|
||||
"FL_HFDatasetDownloader": "FL HF Dataset Downloader",
|
||||
"FL_WF_Agent": "FL Workflow Agent",
|
||||
"FL_BlackFrameReject": "FL Black Frame Reject",
|
||||
"FL_PixVerseAPI": "FL PixVerse API",
|
||||
"FL_Fal_Pixverse": "FL Fal Pixverse API",
|
||||
"FL_Fal_Kontext": "FL Fal Kontext API",
|
||||
"FL_Fal_GPTImage2_Edit": "FL Fal GPT-Image-2 Edit",
|
||||
"FL_Fal_Gemini_ImageEdit": "FL Fal Gemini Image Edit",
|
||||
"FL_Fal_Seedance2_ReferenceToVideo": "FL Fal Seedance 2.0 Reference to Video",
|
||||
"FL_Fal_Seedance_i2v": "FL Fal Seedance i2v",
|
||||
"FL_Fal_Seedream_Edit": "FL Fal Seedream Edit",
|
||||
"FL_Fal_SeedVR_Upscale": "FL Fal SeedVR Upscale",
|
||||
"FL_Fal_Pixverse_Transition": "FL Fal Pixverse Transition",
|
||||
"FL_Fal_Pixverse_LipSync": "FL Fal Pixverse LipSync",
|
||||
"FL_Fal_Kling_AIAvatar": "FL Fal Kling AI Avatar",
|
||||
"FL_Fal_Sora": "FL Fal Sora 2",
|
||||
"FL_PromptBasic": "FL Prompt Basic",
|
||||
"FL_PromptMulti": "FL Prompt Multi",
|
||||
"FL_PromptSelectorBasic": "FL Prompt Selector Basic",
|
||||
"FL_PaddingRemover": "FL Padding Remover",
|
||||
"FL_GPT_Text": "FL GPT Text",
|
||||
"FL_GoogleCloudStorage": "FL Google Cloud Storage Uploader",
|
||||
"FL_StringToLoraName": "FL String To Lora Name",
|
||||
"FL_ModelDifferenceLoraSave": "FL Model Difference to LoRA",
|
||||
"FL_Switch": "FL Switch",
|
||||
"FL_Switch_Big": "FL Switch Big",
|
||||
"FL_PasteByMask": "FL Paste By Mask",
|
||||
"FL_Float": "FL Float",
|
||||
"FL_UnloadModel": "FL Unload Model",
|
||||
"FL_UnloadAllModels": "FL Unload All Models",
|
||||
"FL_ModelInspector": "FL Model Inspector",
|
||||
"FL_VideoTrim": "FL Video Trim",
|
||||
"FL_VideoBatchSplitter": "FL Video Batch Splitter",
|
||||
"FL_VideoCrossfade": "FL Video Crossfade",
|
||||
"FL_VideoCadence": "FL Video Cadence",
|
||||
"FL_WanVaceToVideoMultiRef": "FL Wan Vace To Video Multi Reference",
|
||||
"FL_VideoCadenceCompile": "FL Video Cadence Compile",
|
||||
"FL_FILM": "FL FILM Frame Interpolation",
|
||||
"FL_RIFE": "FL RIFE Frame Interpolation",
|
||||
"FL_GPT_Image1_ADV": "FL GPT Image1 ADV",
|
||||
"FL_ImageBatch": "FL Image Batch",
|
||||
"FL_Hedra_API": "FL Hedra API",
|
||||
"FL_RunwayImageAPI": "FL Runway Image API",
|
||||
"FL_RunwayAct2": "FL Runway Act2",
|
||||
"FL_TextOverlayNode": "FL Text Overlay",
|
||||
"FL_SaveWebM": "FL Save WebM",
|
||||
"FL_ImageCrop": "FL Image Crop",
|
||||
"FL_ImageToMask": "FL Image To Mask",
|
||||
"FL_WanFirstLastFrameToVideo": "FL Wan First Frame Last Frame",
|
||||
"FL_WanVideoContinue": "FL Wan Video Continue",
|
||||
"FL_WanVideoBlender": "FL Wan Video Blender",
|
||||
"FL_WanVideoBlender3Way": "FL Wan Video Blender 3-Way",
|
||||
"FL_WanVideoContinuationBlender": "FL Wan Video Continuation Blender",
|
||||
"FL_ZImageControlNetPatch": "FL Z-Image ControlNet Patch Loader",
|
||||
"FL_LTXVMaskedICLoRAGuide": "FL LTXV Masked IC-LoRA Guide",
|
||||
"FL_SaveRGBAAnimatedWebP": "FL Save RGBA Animated WebP",
|
||||
"FL_Audio_BPM_Analyzer": "FL Audio BPM Analyzer",
|
||||
"FL_Audio_Beat_Prompt_Schedule": "FL Audio Beat Prompt Schedule",
|
||||
"FL_Prompt_Reference_Library": "FL Prompt Reference Library",
|
||||
"FL_Audio_Beat_Prompt_Envelope": "FL Audio Beat Prompt Envelope",
|
||||
"FL_Audio_Envelope_Prompt": "FL Audio Envelope Prompt",
|
||||
"FL_Audio_Beat_Visualizer": "FL Audio Beat Visualizer",
|
||||
"FL_Audio_Crop": "FL Audio Crop",
|
||||
"FL_Audio_Drum_Detector": "FL Audio Drum Detector",
|
||||
"FL_Audio_Envelope_Visualizer": "FL Audio Envelope Visualizer",
|
||||
"FL_Audio_Music_Video_Sequencer": "FL Audio Music Video Sequencer",
|
||||
"FL_Audio_Reactive_Brightness": "FL Audio Reactive Brightness",
|
||||
"FL_Audio_Reactive_Edge_Glow": "FL Audio Reactive Edge Glow",
|
||||
"FL_Audio_Reactive_Envelope": "FL Audio Reactive Envelope",
|
||||
"FL_Audio_Reactive_Saturation": "FL Audio Reactive Saturation",
|
||||
"FL_Audio_Reactive_Scale": "FL Audio Reactive Scale",
|
||||
"FL_Audio_Reactive_Speed": "FL Audio Reactive Speed",
|
||||
"FL_Audio_Segment_Extractor": "FL Audio Segment Extractor",
|
||||
"FL_Audio_Separation": "FL Audio Separation",
|
||||
"FL_Audio_Shot_Iterator": "FL Audio Shot Iterator",
|
||||
"FL_AudioFrameCalculator": "FL Audio Frame Calculator",
|
||||
"FL_QwenImageEditStrength": "FL Qwen Image Edit with Strength",
|
||||
"FL_IntToFloat": "FL Int to Float",
|
||||
"FL_FloatToInt": "FL Float to Int",
|
||||
"FL_AnimatedShapePatterns": "FL Animated Shape Patterns",
|
||||
"FL_PathAnimator": "FL Path Animator",
|
||||
"FL_RandomShapeGenerator": "FL Random Shape Generator",
|
||||
"FL_KartelJobInput": "FL Kartel Job Input",
|
||||
"FL_KartelJobOutput": "FL Kartel Job Output",
|
||||
"FL_ShowText": "FL Show Text",
|
||||
}
|
||||
|
||||
# Conditionally register google-genai nodes if the SDK loaded successfully
|
||||
if _GENAI_NODES_AVAILABLE:
|
||||
NODE_CLASS_MAPPINGS.update({
|
||||
"FL_GeminiImageEditor": FL_GeminiImageEditor,
|
||||
"FL_GeminiImageGenADV": FL_GeminiImageGenADV,
|
||||
"FL_GeminiTextAPI": FL_GeminiTextAPI,
|
||||
"FL_Veo3VideoGen": FL_Veo3VideoGen,
|
||||
"FL_VertexGemini25FlashImage": FL_VertexGemini25FlashImage,
|
||||
})
|
||||
NODE_DISPLAY_NAME_MAPPINGS.update({
|
||||
"FL_GeminiImageEditor": "FL Gemini Image Editor",
|
||||
"FL_GeminiImageGenADV": "FL Gemini Image Gen ADV",
|
||||
"FL_GeminiTextAPI": "FL Gemini Text API",
|
||||
"FL_Veo3VideoGen": "FL Vertex Veo3",
|
||||
"FL_VertexGemini25FlashImage": "FL Vertex Gemini 2.5 Flash Image",
|
||||
})
|
||||
|
||||
|
||||
|
||||
ascii_art = """
|
||||
|
||||
███╗ ███╗ █████╗ ██████╗██╗ ██╗██╗███╗ ██╗███████╗
|
||||
████╗ ████║██╔══██╗██╔════╝██║ ██║██║████╗ ██║██╔════╝
|
||||
██╔████╔██║███████║██║ ███████║██║██╔██╗ ██║█████╗
|
||||
██║╚██╔╝██║██╔══██║██║ ██╔══██║██║██║╚██╗██║██╔══╝
|
||||
██║ ╚═╝ ██║██║ ██║╚██████╗██║ ██║██║██║ ╚████║███████╗
|
||||
╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚═╝╚═╝ ╚═══╝╚══════╝
|
||||
|
||||
██████╗ ███████╗██╗ ██╗ ██╗███████╗██╗ ██████╗ ███╗ ██╗███████╗
|
||||
██╔══██╗██╔════╝██║ ██║ ██║██╔════╝██║██╔═══██╗████╗ ██║██╔════╝
|
||||
██║ ██║█████╗ ██║ ██║ ██║███████╗██║██║ ██║██╔██╗ ██║███████╗
|
||||
██║ ██║██╔══╝ ██║ ██║ ██║╚════██║██║██║ ██║██║╚██╗██║╚════██║
|
||||
██████╔╝███████╗███████╗╚██████╔╝███████║██║╚██████╔╝██║ ╚████║███████║
|
||||
╚═════╝ ╚══════╝╚══════╝ ╚═════╝ ╚══════╝╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚══════╝
|
||||
|
||||
"""
|
||||
try:
|
||||
print(ascii_art)
|
||||
except UnicodeEncodeError:
|
||||
print("[comfyui_fill-nodes] Loaded successfully.")
|
||||
|
||||
WEB_DIRECTORY = "./web"
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
from .fl_image_randomizer import FL_ImageRandomizer
|
||||
from .fl_image_caption_saver import FL_ImageCaptionSaver
|
||||
from .fl_image_dimension_display import FL_ImageDimensionDisplay
|
||||
#from .fl_audio_frame_calculator import FL_AudioFrameCalculator
|
||||
#from .fl_audio_preview import FL_AudioPreview
|
||||
#from .fl_image_duration_sync import FL_ImageDurationSync
|
||||
#from .fl_vhs_audio import FL_AudioConverter
|
||||
from .fl_code_node import FL_CodeNode
|
||||
from .fl_image_pixelator import FL_ImagePixelator
|
||||
from .fl_directorycrawl import FL_DirectoryCrawl
|
||||
from .fl_ascii import FL_Ascii
|
||||
from .fl_glitch import FL_Glitch
|
||||
from .fl_ripple import FL_Ripple
|
||||
from .fl_pixelsort import FL_PixelSort
|
||||
from .fl_hexagonalpattern import FL_HexagonalPattern
|
||||
from .fl_nftgenerator import FL_NFTGenerator
|
||||
from .fl_halftone import FL_HalftonePattern
|
||||
from .fl_randomrange import FL_RandomNumber
|
||||
from .fl_promptselector import FL_PromptSelector
|
||||
from .fl_shader import FL_Shadertoy
|
||||
from .fl_pixelshader import FL_PixelArtShader
|
||||
from .fl_infinitezoom import FL_InfiniteZoom
|
||||
from. fl_paperdrawn import FL_PaperDrawn
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FL_ImageRandomizer": FL_ImageRandomizer,
|
||||
"FL_ImageCaptionSaver": FL_ImageCaptionSaver,
|
||||
"FL_ImageDimensionDisplay": FL_ImageDimensionDisplay,
|
||||
#"FL_AudioPreview": FL_AudioPreview,
|
||||
#"FL_ImageDurationSync": FL_ImageDurationSync,
|
||||
#"FL_AudioConverter": FL_AudioConverter,
|
||||
#"FL_AudioFrameCalculator": FL_AudioFrameCalculator,
|
||||
"FL_CodeNode": FL_CodeNode,
|
||||
"FL_ImagePixelator": FL_ImagePixelator,
|
||||
"FL_DirectoryCrawl": FL_DirectoryCrawl,
|
||||
"FL_Ascii": FL_Ascii,
|
||||
"FL_Glitch": FL_Glitch,
|
||||
"FL_Ripple": FL_Ripple,
|
||||
"FL_PixelSort": FL_PixelSort,
|
||||
"FL_HexagonalPattern": FL_HexagonalPattern,
|
||||
"FL_NFTGenerator": FL_NFTGenerator,
|
||||
"FL_HalftonePattern": FL_HalftonePattern,
|
||||
"FL_RandomNumber": FL_RandomNumber,
|
||||
"FL_PromptSelector": FL_PromptSelector,
|
||||
"FL_Shadertoy": FL_Shadertoy,
|
||||
"FL_PixelArtShader": FL_PixelArtShader,
|
||||
"FL_InfiniteZoom": FL_InfiniteZoom,
|
||||
"FL_PaperDrawn": FL_PaperDrawn
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_ImageRandomizer": "FL Image Randomizer",
|
||||
"FL_ImageCaptionSaver": "FL Image Caption Saver",
|
||||
"FL_ImageDimensionDisplay": "FL Image Size",
|
||||
#"FL_AudioPreview": "FL Audio Preview",
|
||||
#"FL_ImageDurationSync": "FL Image Duration Sync",
|
||||
#"FL_AudioConverter": "FL VHS Audio Converter",
|
||||
#"FL_AudioFrameCalculator": "FL Audio Scanner",
|
||||
"FL_CodeNode": "FL Code Node",
|
||||
"FL_ImagePixelator": "FL Image Pixelator",
|
||||
"FL_DirectoryCrawl": "FL Directory Crawl",
|
||||
"FL_Ascii": "FL Ascii",
|
||||
"FL_Glitch": "FL Glitch",
|
||||
"FL_Ripple": "FL Ripple",
|
||||
"FL_PixelSort": "FL PixelSort",
|
||||
"FL_HexagonalPattern": "FL Hexagonal Pattern",
|
||||
"FL_NFTGenerator": "FL NFT Generator",
|
||||
"FL_HalftonePattern": "FL Halftone",
|
||||
"FL_RandomNumber": "FL Random Number",
|
||||
"FL_PromptSelector": "FL Prompt Selector",
|
||||
"FL_Shadertoy": "FL Shadertoy",
|
||||
"FL_PixelArtShader": "FL Pixel Art",
|
||||
"FL_InfiniteZoom": "FL Infinite Zoom",
|
||||
"FL_PaperDrawn": "FL Paper Drawn"
|
||||
}
|
||||
|
||||
|
||||
ascii_art = """
|
||||
|
||||
███╗ ███╗ █████╗ ██████╗██╗ ██╗██╗███╗ ██╗███████╗
|
||||
████╗ ████║██╔══██╗██╔════╝██║ ██║██║████╗ ██║██╔════╝
|
||||
██╔████╔██║███████║██║ ███████║██║██╔██╗ ██║█████╗
|
||||
██║╚██╔╝██║██╔══██║██║ ██╔══██║██║██║╚██╗██║██╔══╝
|
||||
██║ ╚═╝ ██║██║ ██║╚██████╗██║ ██║██║██║ ╚████║███████╗
|
||||
╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝ ╚═╝╚═╝╚═╝ ╚═══╝╚══════╝
|
||||
|
||||
██████╗ ███████╗██╗ ██╗ ██╗███████╗██╗ ██████╗ ███╗ ██╗███████╗
|
||||
██╔══██╗██╔════╝██║ ██║ ██║██╔════╝██║██╔═══██╗████╗ ██║██╔════╝
|
||||
██║ ██║█████╗ ██║ ██║ ██║███████╗██║██║ ██║██╔██╗ ██║███████╗
|
||||
██║ ██║██╔══╝ ██║ ██║ ██║╚════██║██║██║ ██║██║╚██╗██║╚════██║
|
||||
██████╔╝███████╗███████╗╚██████╔╝███████║██║╚██████╔╝██║ ╚████║███████║
|
||||
╚═════╝ ╚══════╝╚══════╝ ╚═════╝ ╚══════╝╚═╝ ╚═════╝ ╚═╝ ╚═══╝╚══════╝
|
||||
|
||||
|
||||
"""
|
||||
print(ascii_art)
|
||||
|
||||
WEB_DIRECTORY = "./web"
|
||||
__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,21 +0,0 @@
|
||||
# Timeline references and storyboards
|
||||
|
||||
All storyboards requested in one Writer response are submitted in one graph of independent asynchronous Nano nodes, without a local concurrency-count cap. Queue items still execute normally; provider throttling, request-size validation and shared batch completion affect actual speed. Large requests may spend credits simultaneously. There are no automatic paid retries.
|
||||
|
||||
Timeline and moodboard images use local 256-pixel WebP thumbnails with cache revalidation. Originals load only in the image popup, alongside available generation metadata. Reroll/remove controls sit below each timeline image. Loading indicators distinguish thumbnail loading from queued/generating jobs; reduced-motion preferences are respected. Wheel gestures over reference cards zoom/pan the timeline normally.
|
||||
|
||||
All sections requested in a Writer response share the same continuity brief and snapshot of checked moodboards. For stronger consistency, supply a clear character sheet and a style reference with explicit roles. Text alone cannot guarantee identical designs. Rerolls reuse the original brief and moodboard references; ask the Writer for new storyboards to use changed moodboard selections. A first generated master reference can be uploaded as a checked moodboard for subsequent sections; automatic master-reference generation is not implemented.
|
||||
|
||||
Ask Beat Writer to generate storyboards for the selected sections. It writes chronological image prompts and queues official Nano Banana 2 Partner Node jobs in ComfyUI. Prefer 2×2 grids for sections up to six seconds, 3×3 for longer sections.
|
||||
|
||||
Completed sheets are split into panels locally and assigned automatically. Active sheet thumbnails on timeline clips represent selected video references. If a section changes or attachment fails, its saved sheet remains visible as **not attached**, with a reason and an **Attach** button that uses no generation credits. **×** dismisses an unattached result without changing active references. Generated storyboards take precedence over reference assignments for the same section in one Writer response. Click a thumbnail to preview; **↻** generates a replacement. The old references remain active while a reroll runs. Generation and rerolls use ComfyUI credits without another approval dialog.
|
||||
|
||||
Four moodboard slots accept uploads, drops and pasted images. Add an optional role and check **Use** to send that image to the Writer and Google's image-generation endpoint. Unchecked images stay local. Checked moodboards and chat attachments share the Writer's eight-image limit.
|
||||
|
||||
The scheduler must connect directly to **FL MiniMax H3 Beat Shot Planner**. Storyboard generation connects **FL Prompt Reference Library** automatically and sets the planner's visual reference mode to **full**. Custom section references replace the planner's default media. The timeline soundtrack is unchanged. Panels are visual references, not exact timed keyframe constraints.
|
||||
|
||||
The optional **Section references** inspector supports manual image, video and audio assignments. `<Picture N>` and `<Video N>` each start at 1 within their media type, in selection order. Video audio remains paired; audio numbering also includes the timeline song. Only selected files are loaded and encoded. Grouped render sections must share the same ordered references.
|
||||
|
||||
Leave the editor open while the Writer prepares image requests. Submitted image jobs continue in ComfyUI if the editor closes; reopening recovers their results. Edited or deleted sections do not receive stale results automatically. Reconnecting cannot repeat an already submitted paid request. Check ComfyUI history for failed or unknown submissions before generating again; cancellation does not guarantee a refund.
|
||||
|
||||
Workflows store metadata, not embedded media. Share the referenced files under `input/fl-prompt-references`, `input/fl-beat-writer` and `output/fl-storyboards` with the workflow, preserving relative paths. Job records stay in the local Beat Writer data directory. Credentials are not stored in these manifests.
|
||||
@@ -1,15 +0,0 @@
|
||||
# Dynamic Scan FX analysis
|
||||
|
||||
Reference-file change detection belongs to the scheduler's serialized reference settings. The reference library no longer fingerprints its connected schedule: ComfyUI does not supply linked values during fingerprinting, which previously raised a warning and invalidated downstream caches on every run. Selected reference-file path, modification time and size still invalidate the scheduler when changed.
|
||||
|
||||
Connect a video batch and its FL prompt schedule to **FL Scan Video Shots**. Its IMAGE list runs one shared depth/normals/detection/pose branch for every authored shot. Adjacent sections with the same render group stay together. Without a schedule the complete video is one shot. A supplied schedule must match the video's frame count and cover it without gaps or overlaps.
|
||||
|
||||
Package each result with **FL Scan Analysis**, then connect **FL Scan Analysis Collect** to one Interactive Scan FX analysis socket. The collector preserves order and shot boundaries without concatenating analysis tensors. Existing manually connected analysis sockets still work. Processing follows actual shot boundaries, not arbitrary quarters; this changes effect resets compared with the old four-quarter layout.
|
||||
|
||||
For VFX-only iteration, use an **FL Switch** with the saved raw video on `on_false` and the H3 assembler on `on_true`. Set it to false, and disable any separate output nodes that still request the H3 branch. The saved video is an explicit input, not a cache that automatically updates when prompts change. Select a new saved take after generating one; retain its matching schedule and FPS.
|
||||
|
||||
ComfyUI's RAM-pressure cache can evict analysis results. On builds with active/inactive headroom arguments, `--cache-ram 10 16` retains results when memory permits while keeping headroom. These are free-RAM thresholds in GB, not cache-size limits. Keep the flags in the launcher used for subsequent starts. A saved source prevents H3 reruns even after cache eviction or restart; analysis may still rerun.
|
||||
|
||||
The synthetic Scan FX demo renders at 320 pixels and up to 12 FPS, applies color filters once per surface, and stops animation scheduling when paused, offscreen, inactive, or in a hidden tab. These settings do not alter final video rendering.
|
||||
|
||||
Voxel polygon coordinates and face colors are prepared in arrays before rasterization, preserving painter order and OpenCV drawing. Saturation and edge glow use ComfyUI's selected compute device one frame at a time, including transfers back to CPU output. Low GPU-memory headroom selects CPU instead. Brightness-mask preparation, projection, optical flow and cursor composition remain CPU-based. GPU arithmetic may differ from CPU by floating-point roundoff; this is not a fully GPU-rendered compositor.
|
||||
@@ -1,25 +0,0 @@
|
||||
# Dynamic parallax posters
|
||||
|
||||
`FL Image Layer Planner` (existing node ID `FL_PosterLayerPlanner`) inspects one finished image using a vision-capable CLIP, or accepts a manual JSON layer list. Auto mode chooses up to the requested budget, including one filled background. Grouping controls whether related objects stay together; `artwork_type` guides photography, illustration, graphic design, products and mixed media. Text layers are only planned when lettering is visible. Auto can correct an invalid plan once and removes other-object location clauses from cutout targets; Manual prompts remain unchanged. Typography is extracted from the generated artwork, not rendered as an overlay.
|
||||
|
||||
Manual plans use this form:
|
||||
|
||||
```json
|
||||
[
|
||||
{"name":"Paper", "kind":"background", "prompt":"Cream paper without objects or lettering", "depth":12},
|
||||
{"name":"Headline", "kind":"text", "prompt":"The large black AFTER HOURS lettering at the top", "depth":8},
|
||||
{"name":"Hero", "kind":"art", "prompt":"Woman wearing a silver jacket and headphones", "depth":4}
|
||||
]
|
||||
```
|
||||
|
||||
`FL Poster Layers` expands the plan into native ComfyUI conditioning, sampling, and decode nodes. Use **Qwen Image Layered Control**, its text encoder, and the layered RGBA VAE. The image reference is encoded once; the Control latent has `layers=0`, meaning one RGBA result per extraction. Each layer has its own stable sampler ID and seed.
|
||||
|
||||
The review panel shows small thumbnails and full-resolution RGBA on click. Edit a basic-English extraction prompt, depth, scale, offset or visibility, then Run. Reroll changes one layer's seed and queues the workflow. Reset removes that layer's edits for the next Run. Edits are scoped to the plan; changed plans do not inherit stale object prompts.
|
||||
|
||||
Connect the resulting `FL_PARALLAX_STACK` to **FL Layered Parallax**. It supplies the background and a variable-length list of cutouts. Existing individual layer inputs still work. Lower depths are nearer and move faster; keep the camera background depth above every cutout depth.
|
||||
|
||||
The compositor now includes [Parallax Studio](parallax_editor.md), a visual camera editor with live low-resolution previews, presets, drag controls and cover/contain framing. Width, height and frames remain standard widgets.
|
||||
|
||||
Layout changes only affect stack assembly/composition. Prompt edits or rerolls invalidate the affected extraction path, not every sampler. A node-scoped cache provider recovers exact RGBA tensors when ComfyUI's RAM cache evicts them. It uses ComfyUI's native dependency signatures, writes safetensors atomically under `user/__cache/fl_poster_layers_v1`, and keeps at most 1 GB by removing its oldest cache files. No large tensor store is retained by the provider. Clearing that folder forces fresh extraction; exported PNGs remain untouched. RGBA assets and 192-pixel thumbnails are saved under `output/Dynamic_Parallax_Poster/layers` and `thumbnails`, so saved reviews survive a server restart.
|
||||
|
||||
These models generate their extraction: lettering can change, small items can be omitted, and hidden regions can be invented. Check the source spelling, alpha previews, and reassembled still before exporting motion. Auto planning is not a guarantee of perfect semantic separation.
|
||||
@@ -1,80 +0,0 @@
|
||||
# Gemini media inputs and music captioning
|
||||
|
||||
FL Gemini Text API accepts optional `audio` (AUDIO), `image` (IMAGE), and `video` (native VIDEO) connections, together or separately. Its response remains a STRING, and existing text-only workflows still work. Use a Google API key on the node. The selected Gemini model must support the connected media; the custom model field accepts an explicit model ID, subject to Google's availability and capabilities.
|
||||
|
||||
Connect Load Audio to `audio`, Load Image to `image`, or Load Video/Create Video to `video`. Every audio/image batch item is sent in order as a separate attachment. An image batch has no frame timing or soundtrack; use VIDEO for timed footage with sound. Native videos are exported through ComfyUI's video interface, including the selected trim and audio.
|
||||
|
||||
Media is encoded into temporary files, uploaded through Google's Files API, and deleted remotely after the request, including on failures. Local temporary files and the API client are also cleaned up. File processing has a five-minute deadline. Large files remain subject to the selected model's media and context limits. This node does not silently shorten recordings. The output is text; asking for JSON in the prompt does not enforce a JSON schema.
|
||||
|
||||
## What to capture for YuE2
|
||||
|
||||
Prioritize audible properties that you would want to request during generation:
|
||||
|
||||
| Property | Useful detail |
|
||||
| --- | --- |
|
||||
| Rhythm | Half-time or four-on-the-floor feel, swing, syncopation, kick/snare placement, percussion density. Distinguish pulse feel from estimated BPM. |
|
||||
| Bass | Sub weight, sustained versus rhythmic notes, wobble or modulation, call-and-response with drums. |
|
||||
| Instruments and motifs | Recurring melodic hook, chord stabs, pads, instrumental roles and how a motif changes. |
|
||||
| Development | Order of sections, additions and removals, contrast between drops, breakdown length, return of a motif. |
|
||||
| Energy and transitions | Sparse versus dense passages, tension/release, fills, risers, abrupt cuts, gradual filtering. |
|
||||
| Space and texture | Dry drums, long dub-delay tails, wide pads, gritty bass, restrained or bright high end. Describe audible effects without inventing hardware or settings. |
|
||||
| Vocals | Sung/spoken/chopped texture, delivery, language, layering, presence of intelligible words. |
|
||||
| Harmony | Stable or changing harmony, tonal mood; exact key/chords only when reliable. |
|
||||
|
||||
For progression, prefer relationships: “The second drop adds a higher bass response and denser hats” is more informative than “energetic, dark, atmospheric.” Include steady, repetitive arrangements when that is what the recording actually does.
|
||||
|
||||
Keep training captions compact and consistent across recordings. A practical starting point is roughly 60–120 words, adjusted to the material; this is a captioning convention, not a YuE2 requirement. Avoid artist identification, praise, invented lyrics, and unsupported technical precision.
|
||||
|
||||
## Separate training text from analysis
|
||||
|
||||
YuE2's current dataset uses `.caption.txt` for style and `.lyrics.txt` for lyrics. Its AR training request uses planning off. A timestamped arrangement map is not currently read as a training control or converted into a score. Describing an arrangement in the style caption can supply useful conditioning, but does not guarantee exact timed events or train score planning.
|
||||
|
||||
Use three outputs from an analysis prompt:
|
||||
|
||||
1. **Style:** a concise description of the sound, plus a short chronological account of the arrangement. Copy this field into the reviewed style caption.
|
||||
2. **Lyrics:** exact audible words, preserving repetitions and justified section labels. Leave empty for instrumental recordings. Keep production notes out of this field.
|
||||
3. **Timeline and uncertainty:** approximate section boundaries, audible changes, confidence, and passages needing review. Keep these as separate analysis metadata.
|
||||
|
||||
If training on extracted excerpts, caption what is audible in each excerpt. A caption describing a full-song buildup and second drop is misleading when paired with a clip containing only the intro. When splitting songs, retain their shared `.song.txt` identity so related excerpts stay together in YuE2's train/validation split.
|
||||
|
||||
## Analysis prompt for FL Gemini Text API
|
||||
|
||||
Use audio alone when creating factual audio-training labels. If you also attach cover art or video, explicitly separate visual observations from audible evidence.
|
||||
|
||||
```text
|
||||
Listen to the entire attached recording and describe only audible evidence.
|
||||
Do not identify the artist or use visual content to infer musical properties.
|
||||
Return JSON with these fields:
|
||||
|
||||
style: A compact music-training caption, approximately 60–120 words.
|
||||
Cover genre, rhythm and pulse feel, bass character, instruments and recurring
|
||||
motifs, vocals, mood, spatial texture and production. Then describe the actual
|
||||
progression in order: what enters or leaves, how tension develops, how later
|
||||
sections differ, and how it ends. Do not invent development if it stays steady.
|
||||
|
||||
lyrics: Transcribe all intelligible lyrics verbatim in their original language
|
||||
and order. Preserve every repetition. Use [Verse], [Chorus], [Bridge], etc. only
|
||||
when justified. Do not include arrangement descriptions as lyrics. For an
|
||||
instrumental use an empty string. Do not guess unclear words.
|
||||
|
||||
instrumental: Boolean.
|
||||
|
||||
sections: Array of {start_seconds, end_seconds, label, audible_description,
|
||||
change_from_previous, confidence}. Times are approximate seconds from the
|
||||
beginning of this attachment. Describe audible changes rather than applying
|
||||
a standard pop structure. Do not invent exact beat or bar boundaries.
|
||||
|
||||
uncertainty: List unclear lyric passages and uncertain musical observations.
|
||||
Omit exact BPM, key, chords or technical settings when uncertain.
|
||||
|
||||
complete: Boolean indicating whether you analyzed and transcribed the entire
|
||||
recording. If incomplete, say what was omitted in uncertainty.
|
||||
```
|
||||
|
||||
Review the JSON before copying fields to the YuE2 sidecars. The existing YuE2 captioner has its own stricter output schema; this richer prompt is for the general Gemini node. To improve that captioner's style output now, add an instruction such as:
|
||||
|
||||
> In the style field, describe the arrangement in chronological order. Identify audible entrances, removals, breakdowns, motif changes, and differences between successive drops. Describe steady repetition honestly. Keep timestamps and production notes out of the lyrics field.
|
||||
|
||||
Approximate timestamps are review aids, not forced alignment. Check important boundaries against the audio before using them to create training excerpts.
|
||||
|
||||
References: [Google audio understanding](https://ai.google.dev/gemini-api/docs/audio), [Google video understanding](https://ai.google.dev/gemini-api/docs/video-understanding).
|
||||
@@ -1,89 +0,0 @@
|
||||
# H3 Street Scan
|
||||
|
||||
A local style study inspired by [Ingi Erlingsson's “please hold”](https://x.com/ingi_erlingsson/status/2097056649559134367), using newly generated footage rather than frames from the reference.
|
||||
|
||||
## Audio-reactive Chaos variant
|
||||
|
||||
### Native depth camera and audio mapping
|
||||
|
||||
Interactive Scan FX includes our own CPU depth-parallax renderer; it does not depend on Depthflow, OpenGL, or another model. It extends the existing depth projection with horizontal/vertical camera offsets and dolly, counter-adjusted around a selected steady-depth plane. `current` preserves the original renderer. `depth_parallax` can affect the `whole_scene` or only the voxel/depth layers inside reveals (`reveals_only`). Both remain 2.5D: newly exposed surfaces are filled from visible samples, not reconstructed.
|
||||
|
||||
Reveal weights choose voxel normals, edge scan, or depth **once per gesture** using the seed. Zero disables a layer; the total must remain positive. Depth styles are grayscale, false color, and contours. Equal voxel/edge weights with depth disabled preserve the original selection. Cursor activity thins gestures; reveal size is sampled at gesture start.
|
||||
|
||||
The on-node Audio Mapping panel assigns Kick/Snare/Hat envelopes to supported parameters over frame ranges `[start, end)`. Click a mapping to select its range, drag either timeline edge to resize, or drag its interior to move it. Clicking outside the selected range seeks the preview. Numeric range fields are available too. Overlapping enabled mappings for the same parameter are rejected. Mappings replace the parameter's value inside the range; outside it the base value remains. Brightness, saturation and glow retain their original envelope reactions outside mapped ranges, so mappings do not double-apply them.
|
||||
|
||||
Smoothing is optional, measured in seconds, and resets at shot boundaries. Speed mappings accumulate animation phase without resetting it. Cube size, FPS and seed are not modulation targets. The timeline shows the last rendered values and authored shot boundaries; settings changes require another execution. A Depth diagnostic tab shows the projected map before camera accents. Old workflows retain the original defaults until the new options are enabled.
|
||||
|
||||
The current workflow consolidates the effect into **FL Interactive Scan FX**. Each **FL Scan Analysis** packages a shot's depth, normals and optional masks/detections/pose; connect these in chronological order to the growing shot inputs. Their total frame count and resolution must match the source video, and all three envelopes must match its FPS and frame count. Generation and analysis remain external and cacheable.
|
||||
|
||||
The unified node reuses voxel rendering, digital depth projection, audio-locked cursor editing, masked snare brightness, kick saturation and kick edge glow. Its main widgets control the surface, cursor count, camera motion and seed. The Advanced panel contains projection, cut timing, cursor scale/reveal strength and finishing controls. Existing individual nodes remain available.
|
||||
|
||||
Its silent diagnostic preview has **Final**, **Surface**, **Mask**, and **Debug** tabs, a frame scrubber and synchronized Kick/Snare/Hat meters. Surface shows projection before camera accents; Debug overlays planned gesture trajectories on the final result. Previews describe the last execution, not live parameter changes. They are temporary MP4 files; rerun if ComfyUI clears its temp directory. Full-resolution final/surface/mask outputs and a timing report remain available. The labeled FL Video Combine output retains audio.
|
||||
|
||||
The current Y2K workflow uses `digital_layers` on all four Scan Composite nodes: a full-scene depth projection with filled splat gaps, crisp white borders and offset cobalt/black backing panels. It does not use the eroded fragment mask or temporal edge ghosts. `fragment` remains the default for older workflows; raggedness and echo controls apply only to that mode.
|
||||
|
||||
Frame- and second-based prompt schedules can run with no detected beats. Beat-index schedules still require a beat grid. The current workflow uses scheduler envelopes 1/2/3 as kick/snare/hat controls: every beat, every other beat, and short sixteenth-note pulses. These are authored grid events, not detected instrument stems. Its four 48-frame sections produce 192 frames (8 seconds), using seed 65, 10 SA Solver steps, beta57, CFG 1 and denoise 1. Earlier test settings below describe the original Chaos version.
|
||||
|
||||
**FL Voxel Normal Relief** now sits between Depth to Normals and each Scan Composite. It draws normal-colored cuboids with shaded sides; relative depth sets their height and a seeded smooth wave adds subtle animation. The four shared controls are cube size (12 px), relief (0.65), animation (0.18), and speed (0.7 cycles/second). This is a 2.5D image effect, not a voxel mesh. Keep its depth and normals aligned, and connect it before projection so the cursor reveals follow the same scene and camera transforms. The raw normal previews remain unchanged. FL Video Combine saves the labeled before/after with audio.
|
||||
|
||||
The primary **H3 Street Scan Chaos - Generate and Composite** workflow uses **FL Audio Beat Prompt Schedule** as its audio source, frame-accurate trim and prompt timeline. The test crop is `Inspired by dnb.mp3`, frames 768–1032 at 24 fps (32–43 seconds). Its locally installed Beat This model supplies beat timing.
|
||||
|
||||
The schedule feeds **FL MiniMax H3 Beat Shot Planner**, including its matching audio and reactive prompt-envelope outputs. Each render receives a shot-local audio reference and rebased beat-weighted motion prompts. **Beat KSampler** uses the Turbo LoRA, four Euler steps, simple scheduling and the existing 12/3 video/audio sigma shifts. **Shot Assembler** removes H3 padding. Audio conditioning encourages musical movement; it does not guarantee frame-perfect physical accents.
|
||||
|
||||
**FL Scan Audio Edit** defaults to `audio_locked` in this workflow. It preserves every source frame chronologically and adds abrupt camera cuts, punch-ins and tilt without retiming generated motion. Its backwards-compatible `remix` mode still permits random seeks and playback-rate changes; those intentionally discard diffusion-time alignment. Keep the four analysis slices and shot-length settings consistent with the authored prompt sections when changing section boundaries.
|
||||
|
||||
Cursor gestures start independently on audio accents rather than restarting at every cut. A fixed seed controls their positions, directions, curved paths, sizes, durations and normal/edge layer choices. Silence produces no new gestures. The report includes cursor events and exact source-frame mapping. The reveal mask excludes cursor graphics and can drive downstream masked effects.
|
||||
|
||||
The existing Fill reactive brightness, saturation and edge-glow nodes provide additional modulation. The current workflow no longer uses Drum Detector.
|
||||
|
||||
The older Chaos FX Lab remains a cached-footage remix workflow. The primary workflow no longer depends on those clips. All Chaos outputs are saved under `output/FL_H3_StreetScan_Chaos`.
|
||||
|
||||
## Workflows
|
||||
|
||||
- **H3 Street Scan - Generate and Composite**: four editable H3 prompts, native Turbo sampling, analysis, effects and assembly.
|
||||
- **H3 Street Scan - FX Lab**: the same effects applied to saved H3 clips. Use this for fast look development or replace the four video paths with your own footage.
|
||||
|
||||
Both are saved in `user/default/workflows`. Rendered videos and workflow copies are in `output/FL_H3_StreetScan`.
|
||||
|
||||
Four 66-frame sections at 24 fps produce an 11-second, 640×640 silent montage. The comparison output places the original on the left and the scan treatment on the right. FX Lab paths refer to local generated clips; include those clips and update paths when sharing. The full generation workflow does not depend on those cached files.
|
||||
|
||||
## Pipeline
|
||||
|
||||
H3 generates full-frame footage. Depth Anything V2 estimates relative depth; blurred depth feeds Depth to Normals. Impact/Ultralytics supplies person segmentation and face/hand detections. DWPose supplies body and hand joints. `FL Scan Video Detections` matches box IDs between adjacent frames. `FL Street Scan Composite` applies depth reprojection, a chipped scene boundary, normal-map flashes, tracking graphics, pose flashes and short echoes.
|
||||
|
||||
This is **2.5D reprojection**, not a multi-view reconstruction or exportable scene mesh. Depth annotations are normalized relative estimates, not metric coordinates. Tracking IDs are nearest-neighbor associations, not persistent identity recognition. The compositor accepts DWPose's pixel-coordinate `POSE_KEYPOINT` batches; pose input is optional.
|
||||
|
||||
Each shot is analyzed separately so depth smoothing, feature tracking and echo history reset at cuts. Source, depth, normals, masks, detections and pose must describe the same frames and resolution (pose includes its own canvas dimensions).
|
||||
|
||||
## Controls
|
||||
|
||||
- **Orbit / depth relief:** parallax. Larger values expose more missing surfaces and reprojection gaps.
|
||||
- **Scene scale / raggedness:** framing and chipped boundary. The detected subject is retained before reprojection.
|
||||
- **Normal mix:** surface-color flashes and the rectangular normal-field panel.
|
||||
- **HUD / pose opacity:** detection graphics and body/hand skeleton flashes.
|
||||
- **Echo strength:** two-frame edge trails.
|
||||
|
||||
Seeds are fixed for repeatable tuning. Depth and normal previews are below each analysis branch. Changing an effect control does not require another H3 generation; Comfy's execution cache can also reuse unchanged analysis while it remains available.
|
||||
|
||||
## Live effect preview
|
||||
|
||||
The shared workflow's four analysis branches use **FL Scan Video Section**, indices 0–3 with `sections=4`, all connected to the same video as Interactive Scan FX. Boundaries are derived from the input frame count, so the chunks cover the complete video exactly once, including uneven remainders. These are processing chunks, not detected scene cuts. Keep all four branches connected; video/envelope FPS and duration must still agree.
|
||||
|
||||
Minimum/maximum cut-frame controls follow each other when crossed. Reversed ranges in older workflows or API requests are sorted before rendering; they do not abort the effect.
|
||||
|
||||
The right-hand editor uses Voxels, Camera, Reveals, Overlay and Finish tabs. Numeric controls have bounded sliders/fields; `?` opens parameter help. Reset restores the base value without removing mappings. Tab reset requires confirmation. Solo changes the demo view explicitly; switching tabs does not.
|
||||
|
||||
Drag a colored Kick/Snare/Hat chip onto a highlighted parameter, or click the chip then the parameter (keyboard: focus the card and press Enter). Click its colored badge for range, smoothing, invert and enable controls. Existing assignments offer replacement or a new unused time range. Overlapping enabled ranges are rejected in the editor and backend. Reset mapping preserves its assigned time section; Remove mapping disconnects it. The timeline below the preview edits the selected badge's range; rendered-value meters show the last execution, not live demo predictions.
|
||||
|
||||
Interactive Scan FX has a two-column editor: visuals on the left and effect controls on the right. **Live demo** is a lightweight synthetic depth scene available before execution; use All, Voxels, Depth, Reveals, Overlay, or Finish to inspect approximate effects. Pause the demo to compare parameter changes at a fixed time. It does not run models, queue work, or predict the exact output. Audio mappings, detection, occlusion and edit timing still require a render. **Last render** retains the actual diagnostic video and envelope timeline. Connected parameter inputs take precedence over stored controls; the demo uses stored values. Changes still require execution to update output videos.
|
||||
|
||||
## Installed dependencies used
|
||||
|
||||
- ComfyUI native MiniMax H3 and FL MiniMax H3 LoRA Block Loader.
|
||||
- H3 `minimax_h3_ref2va_pruned_int8_convrot.safetensors`, Qwen3-VL H3 NVFP4 text encoder, H3 video/audio VAEs, and the ref2v Turbo 4-step LoRA.
|
||||
- DepthAnythingV2: `depth_anything_v2_vitl_fp32.safetensors` (bf16 compute).
|
||||
- Impact Pack/Subpack: `person_yolov8m-seg.pt`, `face_yolov8m.pt`, `hand_yolov8s.pt`.
|
||||
- ControlNet Aux DWPose: `yolox_l.torchscript.pt`, `dw-ll_ucoco_384_bs5.torchscript.pt`.
|
||||
- Image Filters (Depth to Normals), KJNodes (side-by-side concatenation), VideoHelperSuite, and Fill Nodes.
|
||||
|
||||
No paid generation API or new model download was used for this test. The named third-party loader nodes may download missing weights on another installation.
|
||||
@@ -1,115 +0,0 @@
|
||||
# FL Krea Reference Control — proposed scope
|
||||
|
||||
Status: implemented as FL Krea Reference and FL Krea Reference Guider. See [usage](krea-reference-control.md). Visual validation results are recorded separately; the initial scope below describes the design and acceptance targets.
|
||||
|
||||
Implementation finding: independent full-image prediction blending still copied source portraits and panel layouts in the three-seed comparison. The delivered node therefore adds Context mode (default), which retains image-informed text states after the vision block, and Full mode, which retains visual tokens as well. Both retain all twelve layers. Context removed the unwanted portraits/panels in that test set; this is not a universal guarantee.
|
||||
|
||||
## Goal
|
||||
|
||||
Generate one coherent image from a scene prompt and independently controlled references. Let the user request the linework from one image, the palette from another, and the subject or composition from another, with adjustable contribution and timing.
|
||||
|
||||
Initial target: the installed Krea 2 Turbo model, Qwen3-VL-4B Krea encoder, and the user's 8-step Euler/simple workflow. Role instructions express intent; they are not trained, disentangled style/identity controls.
|
||||
|
||||
## Findings driving the design
|
||||
|
||||
- Rebalance's `compile_edit` serializes multiple images into one Qwen conversation as `Picture 1`, `Picture 2`, etc. It does not build a pixel grid or independently mix image contributions. A generated grid is a possible model interpretation, not an explicit image assembly operation in this code.
|
||||
- `Krea2EditRebalance` retains only one of twelve conditioning taps. Its complementary reference branch has no effect under the current per-band projection math. This was reproduced with synthetic tensors.
|
||||
- Its private pass cache omits image content and CLIP identity. Different same-size images can return identical cached conditioning. This was also reproduced.
|
||||
- Both existing nodes use vision-language conditioning, without reference latents. The new node must not advertise exact identity, spatial, or pixel preservation on that basis.
|
||||
- Krea already uses ComfyUI's optimized attention. A custom guider can combine model predictions using existing sampling interfaces without replacing attention implementations or changing core model code.
|
||||
|
||||
## User-facing nodes
|
||||
|
||||
Two node types keep individual reference settings beside each image while keeping the sampling controls centralized.
|
||||
|
||||
### FL Krea Reference
|
||||
|
||||
Produces an `FL_KREA_REFERENCE` descriptor containing the image and its interpretation settings. It performs no model encoding itself.
|
||||
|
||||
| Input | Meaning |
|
||||
| --- | --- |
|
||||
| image | One reference image. A batch with more than one image raises a clear error; use existing batch selection/splitting nodes. |
|
||||
| enabled | Exclude this reference completely when disabled. |
|
||||
| role | Style, palette, subject, composition, or custom. Default: style. |
|
||||
| instruction | Optional description of what to borrow and what to leave behind. Editable text supplements the role. |
|
||||
| weight | Nonnegative direct strength of the reference contribution. Zero excludes the reference. Initial neutral value: 1. |
|
||||
| resolution | Longest-side limit: 256, 512, 1024, or 1280. Initial default: 512; preserve aspect ratio. |
|
||||
| start / end | Advanced activation window in diffusion progress, from 0 to 1. Default: full run. |
|
||||
| fade | Advanced fade fraction at each edge of the activation window; capped to prevent overlapping ramps. Default: no fade. |
|
||||
|
||||
No arbitrary four-reference limit. Practical cost is shown in documentation. Initial performance validation covers one, two, and four references.
|
||||
|
||||
Roles generate explicit instructions, not hidden numerical layer presets. For example, style requests medium, linework, shading, and texture while the scene prompt defines the subject. Composition requests broad arrangement without claiming pose locking. Custom uses the user's instruction directly.
|
||||
|
||||
Source cropping uses existing image nodes. Output-region masks and automatic subject extraction are outside the first version.
|
||||
|
||||
### FL Krea Reference Guider
|
||||
|
||||
Inputs: MODEL, CLIP, scene prompt, an optional `io.Autogrow` collection of reference descriptors with minimum zero, and overall reference influence.
|
||||
|
||||
Output: GUIDER, for the existing `SamplerCustomAdvanced` node. Noise seed, sampler, sigma schedule, latent size, and VAE decode stay in standard ComfyUI nodes. No model or image pass-through outputs.
|
||||
|
||||
Overall influence initially spans 0 to 1, with a provisional default of 0.7. Zero is the plain text baseline; one applies the entered reference weights wherever references are fully active. Combined contributions above one extrapolate beyond the baseline.
|
||||
|
||||
The example workflow replaces KSampler with RandomNoise, KSamplerSelect, BasicScheduler, and SamplerCustomAdvanced, retaining Euler/simple, eight steps, fixed seed, and existing latent/decode nodes. This is an intentional sampling interface change; a CONDITIONING-only output cannot provide this independent prediction blend to a stock KSampler.
|
||||
|
||||
## Proposed sampling behavior
|
||||
|
||||
1. Encode the scene prompt as a text-only baseline using Krea's native template.
|
||||
2. Independently encode the same scene prompt with each enabled reference and its role instruction. Each reference branch sees only one image. Preserve all twelve conditioning taps and their metadata.
|
||||
3. At each model evaluation, calculate the baseline prediction and each active reference prediction at the same latent and sigma.
|
||||
4. Combine predictions in their shared latent coordinates. Do not average text or visual embeddings by token index: different sequences do not have reliable token correspondence.
|
||||
|
||||
Let `D0` be the baseline prediction, `Di` each reference prediction, `S` overall influence, `wi` direct enabled weights, and `ei(t)` the reference activation envelope:
|
||||
|
||||
`D = D0 + S * sum(wi * ei(t) * (Di - D0))`
|
||||
|
||||
This is a blend of denoiser predictions, not an image crossfade or a blend of random seeds. The whole reference-conditioned branch contributes, including its role instruction. It is not a mathematically isolated image-only effect.
|
||||
|
||||
Apply positive enabled weights directly, without normalization. Fading one reference therefore reduces its contribution without automatically boosting another. With no references, all weights zero, or S=0, skip reference encoding/evaluation and follow the ordinary text-only CFG-1 path. Skip baseline evaluation when its coefficient is exactly zero. Disabled/zero-weight entries never influence other weights or prompts.
|
||||
|
||||
Schedules use ComfyUI's model sampling conversion from progress boundaries to sigma. Interpolate fades between those converted boundaries, documenting that interpolation occurs in sigma space. Do not count Python callbacks as diffusion steps. Fractional denoise and samplers with multiple evaluations must follow the same sigma-based contract.
|
||||
|
||||
Independent branches remove the shared multi-picture input as a source of panel layout. They cannot guarantee that no grid will be generated, especially when a reference itself contains panels. The effect of weighting on visual style is not guaranteed to be linear or monotonic.
|
||||
|
||||
## Implementation boundaries and cost
|
||||
|
||||
- Add one Python module under `nodes/conditioning/`, register two nodes through the pack's existing mappings, and add focused tests plus an example workflow. Reuse the installed V3 schema patterns for Autogrow.
|
||||
- Use the existing CFGGuider lifecycle and `calc_cond_batch` contract. Register every conditioning branch through the guider so ComfyUI owns hook preparation, devices, loading, offloading, and cleanup.
|
||||
- Evaluate branches sequentially initially and accumulate predictions to avoid retaining N full prediction tensors. No custom attention kernels, backend selection, raw model mutation, extra model copies, dependencies, downloads, or internet access.
|
||||
- Keep the twelve-tap encoder layout intact. Validate the supported Krea model/encoder combination at the node boundary, with an actionable mismatch error.
|
||||
- Scope is CFG-1 Krea Turbo. Additional negative-prompt guidance, other Krea variants, and interactions with guidance-modifying third-party patches need separate validation. Preserve supported standard hooks and report unsupported combinations explicitly.
|
||||
- Worst-case denoiser work is N+1 evaluations per sampler evaluation: four references plus the baseline can cost approximately five evaluations. Wall-clock time and VRAM must be measured, not inferred from this count alone.
|
||||
- No private persistent tensor cache. Ordinary ComfyUI graph caching applies; changing settings that invalidate the guider can initially re-encode its references. Future encoding reuse requires a separate measured need and correct ownership/invalidation.
|
||||
- The descriptor and guider outputs may hold their execution data under normal graph caching. Do not introduce an additional store that retains images, embeddings, or predictions across executions.
|
||||
|
||||
## Prototype and acceptance gates
|
||||
|
||||
First establish whether independent prediction blending visibly improves reference mixing. Do this before polishing controls or adding presets.
|
||||
|
||||
Use a small fixed-seed comparison set: the current dog/illustration case, two clearly different visual styles, palette plus style, subject plus style, and references that contain panels. Compare native text-only, current joint Encode Rebalance, and the proposed blend, holding prompt, seed, size, model, and sampler constant. Use at least three seeds for each selected comparison.
|
||||
|
||||
Review prompt adherence, contribution from both references, unwanted panels/duplicates, source-subject leakage, recognizable requested style features, artifacts, latency, and peak VRAM. Record comparisons for user review; do not label a subjective quality improvement as proven from synthetic tests.
|
||||
|
||||
Required numerical and integration checks:
|
||||
|
||||
- No references and S=0 match the native text-only baseline within the established numerical tolerance.
|
||||
- One reference at S=1 with a full envelope matches that branch alone; the baseline can be skipped.
|
||||
- Removing a zero-weight or disabled reference leaves the result unchanged.
|
||||
- Swapping same-size image contents invalidates upstream execution and changes encoded conditioning when the images encode differently; changing CLIP invalidates encoding as well.
|
||||
- Reordering complete reference descriptors preserves the mixture within floating-point tolerance.
|
||||
- Doubling weights doubles the reference difference from the baseline at a fixed latent and sigma. A lone reference at 0.05 contributes 5% as much as at 1, before overall influence.
|
||||
- Weights, schedules, and short/fractional-denoise runs follow the documented formula. Validate zero-width windows and out-of-range inputs at the boundary.
|
||||
- Multiple differently sized images and prompts with different sequence lengths work without token-index averaging or padding-based correspondence assumptions.
|
||||
- Repeated execution, cancellation, and sampler failure leave no additional model-owned cache or patch behind.
|
||||
- Existing optimized attention, model patches, supported dtype/device handling, and low-VRAM execution continue through ComfyUI's lifecycle. Run available hardware checks; explicitly record untested backends.
|
||||
|
||||
If the blend produces washed-out styles, weak identities, or persistent competing compositions, revisit the conditioning strategy before release. Do not compensate with unexplained layer multipliers or promise a grid-prevention switch.
|
||||
|
||||
## Deferred work
|
||||
|
||||
Per-region destination masks, trained identity preservation, latent-reference control, automatic image captioning, per-layer expert knobs, extrapolating reference strength, a custom canvas editor, and a faster attention-based mode are outside v1. Each needs its own evidence and implementation contract.
|
||||
|
||||
## Deliverables
|
||||
|
||||
Two registered nodes, focused regression tests, a concise usage guide, an example replacing the user's current sampler path, and fixed-seed visual comparisons with timing/VRAM measurements. No existing Rebalance code or user workflow is modified as part of this scope document.
|
||||
@@ -1,41 +0,0 @@
|
||||
# Krea reference control
|
||||
|
||||
Use **FL Krea Reference** for each image and connect its output to **FL Krea Reference Guider**. Connect the guider to **SamplerCustomAdvanced**, with standard noise, sampler, scheduler, and latent nodes. The example uses Krea 2 Turbo, eight steps, Euler/simple, and a fixed seed.
|
||||
|
||||
For **KSampler** or **KSampler Advanced**, connect the guider node's **MODEL** output to `model` and **CONDITIONING** output to `positive`. Both connections are required to preserve independent reference weights and timing. Connect an ordinary empty CLIP Text Encode to `negative`. Use CFG **1** to match the GUIDER output; higher CFG applies normal negative guidance after the reference blend. The original GUIDER output remains available. This changes sampler compatibility, not the reference blending method or its potential for overlapping subjects.
|
||||
|
||||
The guider encodes each reference separately, preserves all twelve Krea conditioning layers, and blends model predictions during sampling. It does not place the reference images into a shared picture grid.
|
||||
|
||||
## Controls
|
||||
|
||||
- **Role:** select what to borrow. Style targets medium and texture; palette targets colors; subject targets appearance; composition targets arrangement. Custom uses only the main scene prompt. These are model instructions, not guaranteed identity or geometry controls.
|
||||
- **Weight:** direct reference strength. 0.05 applies 5% of that reference?s contribution before overall influence; 1 applies the full contribution. Zero or disabled excludes it. Weights are not normalized against other references.
|
||||
- **Influence:** overall reference strength after blending. Zero gives the plain scene prompt. Start at 0.7 and compare using the same seed.
|
||||
- **Blend mode / average amount:** Add preserves the additive blend. In Average mode, amount 0 is additive and amount 1 divides contributions by the number of enabled references; intermediate amounts blend those results. Four enabled references at weight 1 contribute 25% each at full averaging. Individual weights still apply. Disabled references are excluded from the count; zero-weight or faded references retain their share, which returns to the scene prompt. Existing workflows default to Add.
|
||||
- **Resolution:** reference longest-side limit. Start at 512; use 1024 for fine detail. Larger settings increase encoding cost. Images are not enlarged except to meet the minimum vision patch dimensions.
|
||||
- **Reference mode:** Context (default) lets Qwen see the image, then supplies only image-informed text states to Krea. This reduces direct copying of the source layout. Full supplies visual tokens as well, for stronger source resemblance. Both preserve all twelve conditioning layers.
|
||||
- **Start / end / fade:** advanced timing controls in full diffusion progress. Start must be below end. Fade is a fraction of the window at each edge, up to 0.5. Fade interpolation occurs between converted sigma boundaries. Shortened denoise runs use the corresponding part of this full schedule.
|
||||
|
||||
Reference sockets grow as you connect them, up to ComfyUI's 100-entry Autogrow ceiling. Each reference accepts one RGB image; select a frame first when the source is an image batch.
|
||||
|
||||
For a dog rendered like an ink illustration, describe the scene in the main prompt and select a style reference. A second palette reference can supply its colors independently.
|
||||
|
||||
## Cost and behavior
|
||||
|
||||
Each active reference normally adds one model evaluation per sampling evaluation. Two references plus the text baseline require three evaluations. The baseline evaluation is skipped when the combined active reference contribution is exactly 1. Timed-out branches are skipped. This is slower than encoding all pictures together; it makes their numerical contributions independent.
|
||||
|
||||
Weights directly scale each reference?s difference from the scene baseline. Fading one reference returns its contribution to the scene baseline without boosting the other references. The blend is:
|
||||
|
||||
`baseline + influence * blend_scale * sum(weight * envelope * (reference_prediction - baseline))`
|
||||
|
||||
`blend_scale` is 1 in Add mode. In Average mode it is `1 - average_amount + average_amount / N`, where `N` is the number of enabled references. No enabled references uses the baseline. Both GUIDER and MODEL/CONDITIONING outputs use this blend.
|
||||
|
||||
When active weights multiplied by influence sum above 1, the guider extrapolates beyond the baseline instead of silently normalizing or clipping the sliders. Lower overall influence or the reference weights for a gentler result.
|
||||
|
||||
The visual result is nonlinear. A larger reference weight does not guarantee proportionally stronger style. References containing panels can still suggest panels, especially in Full mode, and unrelated subject references can compete. Compare a reference alone before mixing it with others. Context can weaken source identity and geometry; use Full when those matter more than avoiding copied layouts.
|
||||
|
||||
The nodes use normal ComfyUI execution caching, with no private image/embedding cache. Changes to reference settings can re-encode references. Changing same-size image contents or the text encoder is not hidden by a shape-only cache.
|
||||
|
||||
Use a Krea 2 model and CLIPLoader type `krea2`. The guider uses CFG 1 and rejects custom CFG/pre-CFG/post-CFG/batch-guidance functions. Ordinary model patches and optimized attention remain managed by ComfyUI. Other guidance or cross-call caching patches have not been validated with independent reference branches.
|
||||
|
||||
This version does not supply reference latents, destination masks, identity training, or automatic captioning. Use existing image nodes to crop a source reference.
|
||||
@@ -1,45 +0,0 @@
|
||||
# Krea reference validation
|
||||
|
||||
Tested locally on 2026-09-11 with Krea 2 Turbo FP8 scaled, Qwen3-VL-4B FP8 scaled, Qwen Image VAE, and an NVIDIA RTX PRO 6000 Blackwell Max-Q. Eight Euler/simple steps, CFG 1.
|
||||
|
||||
## Automated checks
|
||||
|
||||
17 focused unittest cases pass. They cover the weighted prediction formula, zero influence, the single-reference endpoint, reference order and weight scaling, disabled/zero-weight exclusion, repeat sigma evaluations, fade boundaries, schedule independence, input validation, image/CLIP re-encoding, independent image encoding, twelve-layer preservation, Context suffix copying/mask slicing, and schema/model validation.
|
||||
|
||||
The tests use synthetic predictions for numerical invariants. Real server runs additionally exercised the V3 Autogrow inputs, actual model loading, vision encoding, denoising, and VAE decoding.
|
||||
|
||||
The weight-slider regression tests now check a single reference at 0, 0.05, 0.25, 0.5, 1, and 1.5; independent removal; and combined weights above one. Weights directly scale the baseline difference. Earlier versions normalized weights, making a lone reference effectively on/off.
|
||||
|
||||
After the correction, a live-server sweep rendered a single reference at weights 0, 0.05, 0.25, 0.5, and 1 in both Context and Full modes, with a fixed seed and overall influence 0.7. All five decoded RGB outputs were distinct in each mode. Mean absolute RGB differences from weight zero were 0, 10.92, 28.77, 39.35, and 59.59 for Context, and 0, 12.34, 27.55, 50.15, and 85.53 for Full, on the 0–255 channel scale. The inspected low-weight images stayed close to the baseline; higher Full weights increasingly copied source faces/panels. This demonstrates working numerical strength control in the test case, not universally linear visual response. Images are in `output/Krea_Weight_Test/`.
|
||||
|
||||
## Initial fixed-seed visual comparison (before the weight correction)
|
||||
|
||||
The same dog-in-a-park prompt was generated at 512x768 with seeds 193328989555475, 193328989555476, and 193328989555477. References were a monochrome portrait with window panels and a blue/violet illustration.
|
||||
|
||||
Compared native text-only generation, zero reference influence, each reference alone, independent Full-mode blending, the old joint Encode Rebalance node, and the new Context-mode paths. Zero influence produced pixel-identical decoded RGB images to the native baseline for all three seeds.
|
||||
|
||||
The joint encoder copied portraits and panels in all three comparisons. Independent Full-mode blending also retained unwanted source faces and windows. Context-mode style and mixed outputs preserved the dog scene without those unwanted portraits/panels in the inspected set. The palette-only branch visibly changed the color treatment. With style weight 1 and palette weight 0.5, the blend remained closer to the style branch; relative weights are not linear measures of perceptual influence.
|
||||
|
||||
These are observations from a small test set, not evidence of universal style/identity separation or grid prevention. Subject and composition roles require further use-case validation.
|
||||
|
||||
## Timing and memory
|
||||
|
||||
Warm server execution times at 512x768, excluding queue wait:
|
||||
|
||||
| Path | Time |
|
||||
| --- | --- |
|
||||
| Text-only / zero influence | About 1.7 seconds |
|
||||
| One Context reference | 3.3–3.6 seconds |
|
||||
| Two Context references | About 5.0 seconds |
|
||||
| Four Context references | About 8.4 seconds |
|
||||
|
||||
The first Context run incurred model-loading overhead and is excluded from warm timing. A four-reference run and a faded reference window with denoise 0.75 also completed successfully.
|
||||
|
||||
The updated live Krea workflow also completed at 1024x1536 in 28.35 seconds of server execution, using the user's current woman-on-a-park-bench prompt and two current illustration references. The result followed the scene without constructing a reference grid. Saved locally as `output/Krea_Reference_Test/live_workflow_00001_.png`.
|
||||
|
||||
Polling ComfyUI system stats during the warm Context runs observed about 33.7 GiB total device memory in use. This includes resident models and other GPU allocations; it is not incremental per-reference VRAM or an exact peak. The dynamic allocator is not fully represented by PyTorch's reserved-memory counter.
|
||||
|
||||
CUDA was exercised. CPU, ROCm, MPS, DirectML, XPU, NPU, explicit low-VRAM mode, cancellation during sampling, and arbitrary third-party patches were not separately validated. The implementation uses ComfyUI's existing loading, offloading, attention, and guider cleanup paths.
|
||||
## Standard KSampler outputs
|
||||
|
||||
The guider now also outputs a patched MODEL and CONDITIONING for standard samplers. All 20 unit tests pass, including parity for independent weights, reference schedules, CFG, plain conditioning, and preservation of the original model. A real 512x768, eight-step Euler/simple KSampler render with two references and fading matched SamplerCustomAdvanced pixel-for-pixel at CFG 1 and the same seed. This verifies sampler parity, not improved reference composition quality.
|
||||
@@ -1,147 +0,0 @@
|
||||
{
|
||||
"last_node_id": 3,
|
||||
"last_link_id": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "UNETLoader",
|
||||
"pos": [
|
||||
60,
|
||||
80
|
||||
],
|
||||
"size": [
|
||||
440,
|
||||
86
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "MODEL",
|
||||
"type": "MODEL",
|
||||
"links": [
|
||||
1
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"title": "Fine-tune: Dirtyrealism v5",
|
||||
"properties": {
|
||||
"Node name for S&R": "UNETLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Krea\\krea2SATDirtyrealism_v5_fp8.safetensors",
|
||||
"default"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "UNETLoader",
|
||||
"pos": [
|
||||
60,
|
||||
280
|
||||
],
|
||||
"size": [
|
||||
440,
|
||||
86
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "MODEL",
|
||||
"type": "MODEL",
|
||||
"links": [
|
||||
2
|
||||
],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"title": "Base: Krea2 Turbo",
|
||||
"properties": {
|
||||
"Node name for S&R": "UNETLoader"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Krea\\krea2_turbo_fp8_scaled.safetensors",
|
||||
"default"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "FL_ModelDifferenceLoraSave",
|
||||
"pos": [
|
||||
580,
|
||||
130
|
||||
],
|
||||
"size": [
|
||||
470,
|
||||
210
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "finetuned_model",
|
||||
"type": "MODEL",
|
||||
"link": 1
|
||||
},
|
||||
{
|
||||
"name": "base_model",
|
||||
"type": "MODEL",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "lora_path",
|
||||
"type": "STRING",
|
||||
"links": null,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "FL_ModelDifferenceLoraSave"
|
||||
},
|
||||
"widgets_values": [
|
||||
"Krea2/Dirtyrealism_v5_minus_Krea2_Turbo_rank128",
|
||||
128,
|
||||
"auto"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
3,
|
||||
0,
|
||||
"MODEL"
|
||||
],
|
||||
[
|
||||
2,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
1,
|
||||
"MODEL"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {
|
||||
"ds": {
|
||||
"scale": 1,
|
||||
"offset": [
|
||||
0,
|
||||
0
|
||||
]
|
||||
}
|
||||
},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
# Layered parallax
|
||||
|
||||
`FL Parallax Layer` accepts RGBA video or RGB plus a foreground mask (white = opaque). It adds depth, scale, screen-relative offsets and opacity, and produces a single checkerboard matte preview. An explicit mask overrides embedded alpha.
|
||||
|
||||
`FL Layered Parallax` takes an opaque background and expandable layer inputs. It sorts cutouts far-to-near and projects them using camera translation and forward travel. Smaller depth values move faster; the background must be farther than every cutout. Rendering uses premultiplied-alpha bilinear sampling in four-frame chunks on ComfyUI's selected device, with a CPU option. Outputs are the moving composite, the same animation with the camera locked, and a middle-frame depth preview (near = bright).
|
||||
|
||||
Camera controls:
|
||||
|
||||
- `motion`: burst/settle, eased glide, sinusoidal loop, or locked.
|
||||
- `travel_x/y`: camera excursion in output widths/heights at depth 1; projected displacement divides by depth.
|
||||
- `push_in`: forward travel in depth units. Keep every plane in front of the camera.
|
||||
- `overscan`: enlarges only the background to cover movement. Beyond the source boundary, the background edge is extended; hidden scenery is not generated.
|
||||
|
||||
All animated plates must have equal frame counts. Single images are held over the longest animation. Set `frames` to animate still-image plates for a specified duration; zero inherits the source frame count. An explicit length must match any animated inputs. There is no silent trimming or frame-rate conversion. Set FPS in the exporter.
|
||||
|
||||
This is multiplane 2.5D composition, not a reconstructed scene or novel-view synthesis. It cannot expose unseen sides of objects. Alpha quality and independent generated motion affect the result. Loop camera movement does not make independently generated plates loop seamlessly.
|
||||
|
||||
Changing compositor controls does not change upstream generation or matting inputs. ComfyUI may reuse those cached results while resident; cache survival across restarts or memory pressure is not guaranteed. Save raw plates and RGBA assets for later reuse.
|
||||
@@ -1,9 +0,0 @@
|
||||
# FL Model Difference to LoRA
|
||||
|
||||
Connect two **Load Diffusion Model** nodes: the fine-tune to `finetuned_model`, and its original base to `base_model`. Queue **FL Model Difference to LoRA** to extract `fine-tune - base` and save a numbered `.safetensors` file under the first configured LoRA directory. The `filename_prefix` can include a subfolder, such as `Krea2/Dirtyrealism_difference`.
|
||||
|
||||
Rank defaults to 128. Higher ranks preserve more of the weight difference and produce larger files. `auto` uses ComfyUI's selected device; choose `cpu` if the device lacks SVD support or available memory. Extraction processes one parameter at a time, honors weight patches and quantization scales, and leaves the input models unchanged. Biases and other non-matrix parameters are stored as full differences in ComfyUI's LoRA format. No CLIP extraction is performed.
|
||||
|
||||
Apply the exported file to the same base with **Load LoRA (Model Only)** at strength 1. Low-rank compression is approximate, and quantized source models include quantization differences. Logged retained weight energy measures numerical reconstruction, not image quality. Compare generations before replacing a fine-tune in production workflows.
|
||||
|
||||
The included [Krea2 workflow](krea2_difference_lora.json) selects `krea2SATDirtyrealism_v5_fp8` and `krea2_turbo_fp8_scaled`, saving into `loras/Krea2`. Restart ComfyUI after installing the node, then open the workflow.
|
||||
@@ -1,25 +0,0 @@
|
||||
# Parallax Studio
|
||||
|
||||
FL Layered Parallax keeps width, height and frames as standard widgets. Its camera editor writes the existing motion/framing inputs, so saved workflows and connected inputs remain compatible.
|
||||
|
||||
- **Scene:** small first-frame previews of the actual layers. Drag to set camera travel; Play and the scrubber preview a six-second camera cycle. Center shows neutral framing without changing the render settings.
|
||||
- **Depth order:** near-to-far layer diagram. Edit individual depths in the connected layer nodes or extraction review.
|
||||
- **Camera:** Gentle, Reveal, Punchy and Still presets, motion curves, XY control, horizontal/vertical travel and push/pull. Presets change motion only.
|
||||
- **Framing:** background distance, coverage, cutout cover/contain and render device. Contain preserves mismatched cutout aspect ratios. Background edges use clamped source pixels, not generated hidden scenery.
|
||||
- **Relief:** Off, Background, or Background + Artwork, with strength, anchor depth, inversion and smoothing. Text layers from the poster stack remain flat. Zero strength preserves the previous renderer. The small GPU preview responds to strength/anchor/inversion; smoothing is applied on Run. Without WebGL, the preview remains flat.
|
||||
|
||||
Connect the stack to **FL Parallax Depth Sources**, then an image-batch depth estimator, then `depth_maps` on the compositor. Analysis sources use neutral gray behind cutout alpha and preserve stack order: background, then cutouts. A single depth map is accepted for background-only relief. This path supports still-image layers only. The workflow uses the installed Depth Anything V2 FP32 model at a 518-pixel analysis edge; the inference node is separate so camera and relief adjustments can reuse ComfyUI's cached maps.
|
||||
|
||||
Relief is a bounded inverse-depth warp inside each plate, not a mesh reconstruction. Color and premultiplied alpha share the same sampling grid. Plate draw order remains unchanged, and the Depth order output still visualizes flat layer ordering, not inferred per-pixel depth. Keep relief subtle near silhouettes, reflective objects, and narrow depth gaps. Depth on isolated cutouts can be ambiguous, and unseen surfaces are not reconstructed.
|
||||
|
||||
Before source previews are available, the editor displays labeled demo geometry. Landscape, typography and product-shape demos are preview-only. Actual previews use the same plane projection as the renderer, but reduced-resolution images, browser interpolation and frozen source frames mean they are not final-quality video. Upstream changes require Run to refresh the layers. Connected camera inputs are read-only in the editor and show their last rendered values.
|
||||
|
||||
Preview playback is limited to 20 drawn frames per second, starts paused, and stops scheduling animation when the node is offscreen or the browser tab is hidden. Source previews are at most 256 pixels on their longest edge and are saved under `output/Parallax/previews`.
|
||||
|
||||
The renderer computes a static scene's locked comparison once and shares that frame across the output batch. Animated input plates retain per-frame locked comparisons. Only the representative depth-preview frame is composited. No persistent GPU tensor cache is added.
|
||||
|
||||
## Different media
|
||||
|
||||
The Image Layer Planner follows the visible source medium. `artwork_type` guides grouping; it does not restyle the image. Auto mode adds text layers only when lettering is visible, retains actual environments, and can make one correction pass for an invalid or over-budget plan. Manual mode remains available for exact targets.
|
||||
|
||||
Use subtle travel for portraits, photography, reflective products and fine hair. Illustration and collage usually tolerate more separation; landscapes work best as coherent foreground/middle-ground/background groups. Large camera moves need reconstructed hidden regions or oversized source plates. Generated extraction can alter texture, lettering, shadows and transparency; inspect the RGBA layers before exporting.
|
||||
@@ -1,14 +0,0 @@
|
||||
# Scan FX layers
|
||||
|
||||
The **Layers** tab controls the digital back plates and cursor-window overlap. Existing workflows retain four cobalt plates, normal compositing, full reveal opacity, and no fades.
|
||||
|
||||
- **Stack count**: 0–8 plates. Spacing multiplies the original depth-relative offset; X/Y set direction, rotation fans each successive plate, and opacity affects plates and their outlines without fading the video.
|
||||
- **Window order**: newest on top, a preferred effect on top, or seeded random ordering on Snare (Envelope 2 rising through 0.5). Random ranks hold between hits. They do not shuffle every frame.
|
||||
- **Window blend**: normal, screen, or add. This is separate from Finish's glow blend mode. Per-effect opacity multiplies Reveal strength and the existing audio accent.
|
||||
- **Fade in/out**: seconds for revealed content, not cursor graphics or borders. Fade-in starts when dragging begins; fade-out finishes on the gesture's last frame.
|
||||
|
||||
Drag Kick, Snare, or Hat onto spacing, X/Y spread, rotation, stack opacity, or any reveal opacity to assign an envelope. Count, palette, and ordering remain discrete settings. A useful starting mapping is Kick → stack spacing, minimum 0.6, maximum 2, smoothing 0.08 seconds.
|
||||
|
||||
The compact parameter panel uses two columns when space permits, `?` help, numeric ranges in control tooltips, and individual reset buttons. Mapping editors expand across both columns. Render-value meters appear only for assigned parameters.
|
||||
|
||||
**Live demo → Layers** approximates the stack and overlap on a synthetic scene. **All** includes the other effects. Actual envelopes, geometry, timing, and occlusion must still be checked in **Last render → Final**. These effects reuse existing images and maps; they do not run additional inference.
|
||||
File diff suppressed because it is too large
Load Diff
+104
@@ -0,0 +1,104 @@
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
import numpy as np
|
||||
import torch
|
||||
import sys
|
||||
|
||||
class FL_Ascii:
|
||||
def __init__(self):
|
||||
self.spacing_index = 0
|
||||
self.font_size_index = 0
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"spacing": ("INT", {
|
||||
"default": 20,
|
||||
"min": 4,
|
||||
"max": 100,
|
||||
"step": 2,
|
||||
}),
|
||||
"font_size": ("INT", {
|
||||
"default": 20,
|
||||
"min": 4,
|
||||
"max": 100,
|
||||
"step": 2,
|
||||
}),
|
||||
"characters": ("STRING", {
|
||||
"default": "\._♥♦♣MachineDelusions♣♦♥_./",
|
||||
"description": "characters to use"
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "apply_ascii_art_effect"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def apply_ascii_art_effect(self, image: torch.Tensor, spacing: int, font_size: int, characters):
|
||||
batch_size, height, width, channels = image.shape
|
||||
result = torch.zeros_like(image)
|
||||
|
||||
for b in range(batch_size):
|
||||
img_b = image[b] * 255.0
|
||||
img_b = Image.fromarray(img_b.numpy().astype('uint8'), 'RGB')
|
||||
|
||||
# Check if spacing is a list and get the current value
|
||||
if isinstance(spacing, list):
|
||||
if self.spacing_index >= len(spacing):
|
||||
print("Warning: Spacing list index out of range. Using the last value.")
|
||||
self.spacing_index = len(spacing) - 1
|
||||
current_spacing = spacing[self.spacing_index]
|
||||
self.spacing_index = (self.spacing_index + 1) % len(spacing)
|
||||
else:
|
||||
current_spacing = spacing
|
||||
|
||||
# Check if font_size is a list and get the current value
|
||||
if isinstance(font_size, list):
|
||||
if self.font_size_index >= len(font_size):
|
||||
print("Warning: Font size list index out of range. Using the last value.")
|
||||
self.font_size_index = len(font_size) - 1
|
||||
current_font_size = font_size[self.font_size_index]
|
||||
self.font_size_index = (self.font_size_index + 1) % len(font_size)
|
||||
else:
|
||||
current_font_size = font_size
|
||||
|
||||
result_b = ascii_art_effect(img_b, current_spacing, current_font_size, characters)
|
||||
result_b = torch.tensor(np.array(result_b)) / 255.0
|
||||
result[b] = result_b
|
||||
|
||||
# Update the print log
|
||||
progress = (b + 1) / batch_size * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress log
|
||||
print()
|
||||
|
||||
return (result,)
|
||||
|
||||
|
||||
def ascii_art_effect(image: torch.Tensor, spacing: int, font_size: int, characters):
|
||||
chars = characters
|
||||
small_image = image.resize((image.size[0] // spacing, image.size[1] // spacing), Image.Resampling.NEAREST)
|
||||
|
||||
def get_char(value):
|
||||
return chars[value * len(chars) // 256]
|
||||
|
||||
ascii_image = Image.new('RGB', image.size, (0, 0, 0))
|
||||
font = ImageFont.truetype("arial.ttf", font_size)
|
||||
draw_image = ImageDraw.Draw(ascii_image)
|
||||
|
||||
for i in range(small_image.height):
|
||||
for j in range(small_image.width):
|
||||
r, g, b = small_image.getpixel((j, i))
|
||||
k = (r + g + b) // 3
|
||||
draw_image.text(
|
||||
(j * spacing, i * spacing),
|
||||
get_char(k),
|
||||
font=font,
|
||||
fill=(r, g, b)
|
||||
)
|
||||
|
||||
return ascii_image
|
||||
@@ -0,0 +1,73 @@
|
||||
import os
|
||||
import torchaudio
|
||||
import librosa
|
||||
import numpy as np
|
||||
|
||||
|
||||
class FL_AudioFrameCalculator:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"audio_file_path": ("STRING", {
|
||||
"default": "/path/to/audio/file/example.wav",
|
||||
"description": "Absolute path to the audio file"
|
||||
}),
|
||||
"start_bar": ("INT", {
|
||||
"default": 0,
|
||||
"description": "Start bar"
|
||||
}),
|
||||
"bar_count": ("INT", {
|
||||
"default": 8,
|
||||
"description": "Number of bars"
|
||||
}),
|
||||
"fps": ("INT", {
|
||||
"default": 10,
|
||||
"description": "Frames per second"
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("AUDIO", "INT", "INT", "INT")
|
||||
RETURN_NAMES = ("AUDIO", "Frame Count", "BPM", "FPS")
|
||||
FUNCTION = "process_audio"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def process_audio(self, audio_file_path, start_bar, bar_count, fps):
|
||||
if not os.path.isfile(audio_file_path):
|
||||
raise FileNotFoundError(f"Audio file does not exist: {audio_file_path}")
|
||||
|
||||
waveform, sample_rate = torchaudio.load(audio_file_path)
|
||||
|
||||
y_full = waveform.numpy()[0] # Assuming mono audio for simplicity
|
||||
tempo, _ = librosa.beat.beat_track(y=y_full, sr=sample_rate)
|
||||
bpm = int(tempo)
|
||||
|
||||
# Calculate the duration of a single bar in seconds (4 beats per bar)
|
||||
bar_duration = 60 / bpm * 4
|
||||
|
||||
# Calculate start and end times in seconds based on bars
|
||||
start_time = start_bar * bar_duration
|
||||
end_time = start_time + (bar_count * bar_duration)
|
||||
|
||||
# Convert times to sample indices
|
||||
start_sample = int(start_time * sample_rate)
|
||||
end_sample = int(end_time * sample_rate)
|
||||
|
||||
# Extract the selected audio segment
|
||||
audio_segment = waveform[:, start_sample:end_sample]
|
||||
|
||||
# Calculate the number of frames based on FPS and segment duration
|
||||
duration_seconds = end_time - start_time
|
||||
frames_count = int(duration_seconds * fps)
|
||||
|
||||
audio_output = (audio_segment.cpu().numpy(), sample_rate)
|
||||
|
||||
print("-_-_-_-_-_-_-_-_-_-")
|
||||
print(f"BPM: {bpm}")
|
||||
print(f"Start Time: {start_time}")
|
||||
print(f"End Time: {end_time}")
|
||||
print(f"Frame Count: {frames_count}")
|
||||
print("-_-_-_-_-_-_-_-_-_-")
|
||||
|
||||
return (audio_output, frames_count, bpm, fps)
|
||||
@@ -0,0 +1,44 @@
|
||||
import torchaudio
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import torch # Make sure to import torch for checking tensor type
|
||||
|
||||
class FL_AudioPreview:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {"audio_segment": ("AUDIO", {"description": "Tuple of (audio data tensor, sample rate)"})},
|
||||
}
|
||||
FUNCTION = "play_audio_segment"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
OUTPUT_NODE = True
|
||||
RETURN_TYPES = ()
|
||||
|
||||
@classmethod
|
||||
def play_audio_segment(cls, audio_segment):
|
||||
waveform, sample_rate = audio_segment
|
||||
|
||||
# Check if waveform is a PyTorch tensor and convert to numpy array accordingly
|
||||
if isinstance(waveform, torch.Tensor):
|
||||
numpy_waveform = waveform.cpu().numpy()
|
||||
elif isinstance(waveform, np.ndarray):
|
||||
numpy_waveform = waveform
|
||||
else:
|
||||
raise TypeError("Unsupported type for waveform. Expected torch.Tensor or np.ndarray.")
|
||||
|
||||
# Check if the audio is stereo or mono and play accordingly
|
||||
if numpy_waveform.shape[0] > 1:
|
||||
# If stereo, playing the first channel for simplicity
|
||||
sd.play(numpy_waveform[0], sample_rate)
|
||||
else:
|
||||
# If mono, play as is
|
||||
sd.play(numpy_waveform, sample_rate)
|
||||
|
||||
print ("""
|
||||
---------
|
||||
Will resume after audio preview completes
|
||||
---------
|
||||
""")
|
||||
|
||||
sd.wait()
|
||||
return []
|
||||
@@ -0,0 +1,33 @@
|
||||
class AlwaysEqualProxy(str):
|
||||
|
||||
def __eq__(self, other):
|
||||
return True
|
||||
|
||||
def __ne__(self, other):
|
||||
return False
|
||||
|
||||
|
||||
class FL_CodeNode:
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
required = {"code_input": ("STRING", {"multiline": True})}
|
||||
optional = {f"input_{i}": (AlwaysEqualProxy("*")) for i in range(4)}
|
||||
return {"required": required, "optional": optional}
|
||||
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
RETURN_TYPES = tuple(AlwaysEqualProxy("*") for _ in range(4))
|
||||
RETURN_NAMES = tuple(f"output_{i}" for i in range(4))
|
||||
|
||||
FUNCTION = "execute"
|
||||
|
||||
def execute(self, code_input, **kwargs):
|
||||
outputs = {i: None for i in range(4)}
|
||||
|
||||
try:
|
||||
exec(code_input, {"inputs": kwargs, "outputs": outputs})
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Error executing user code: {e}")
|
||||
|
||||
return tuple(outputs[i] for i in range(4))
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import os
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
class FL_DirectoryCrawl:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"directory_path": ("STRING", {"default": ""}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",) # Output a batch of images
|
||||
FUNCTION = "load_image_batch"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def load_image_batch(self, directory_path):
|
||||
if not directory_path:
|
||||
raise ValueError("Directory path is not provided.")
|
||||
|
||||
image_paths = self.crawl_directories(directory_path)
|
||||
if not image_paths:
|
||||
raise ValueError("No images found in the specified directory and its subdirectories.")
|
||||
|
||||
batch_images = []
|
||||
for img_path in image_paths:
|
||||
image = Image.open(img_path)
|
||||
image = ImageOps.exif_transpose(image) # Correct orientation
|
||||
image = image.convert("RGB")
|
||||
image_np = np.array(image).astype(np.float32) / 255.0
|
||||
batch_images.append(image_np)
|
||||
|
||||
batch_images_np = np.stack(batch_images, axis=0) # Create a numpy array batch
|
||||
batch_images_tensor = torch.from_numpy(batch_images_np) # Convert to tensor
|
||||
|
||||
return (batch_images_tensor,)
|
||||
|
||||
def crawl_directories(self, directory):
|
||||
supported_formats = ["jpg", "jpeg", "png", "bmp", "gif", "txt"]
|
||||
image_paths = []
|
||||
for root, dirs, files in os.walk(directory):
|
||||
for file in files:
|
||||
if file.split('.')[-1].lower() in supported_formats:
|
||||
full_path = os.path.join(root, file)
|
||||
image_paths.append(full_path)
|
||||
return image_paths
|
||||
@@ -0,0 +1,82 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from glitch_this import ImageGlitcher
|
||||
import sys
|
||||
|
||||
class FL_Glitch:
|
||||
def __init__(self):
|
||||
self.seed_index = 0
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"glitch_amount": ("FLOAT", {"default": 3.0, "min": 0.1, "max": 10.0, "step": 0.01}),
|
||||
"color_offset": (["Disable", "Enable"],),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 100, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "glitch"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def s2b(self, v):
|
||||
return v == "Enable"
|
||||
|
||||
def glitch(self, images, glitch_amount=1, color_offset="Disable", seed=0):
|
||||
color_offset = self.s2b(color_offset)
|
||||
g = ImageGlitcher()
|
||||
out = []
|
||||
total_images = len(images)
|
||||
|
||||
# Convert seed to a list if it's a single value
|
||||
if not isinstance(seed, list):
|
||||
seed = [seed] * total_images
|
||||
|
||||
for i, image in enumerate(images, start=1):
|
||||
p = self.t2p(image)
|
||||
|
||||
# Get the current seed value
|
||||
current_seed = seed[i - 1]
|
||||
|
||||
# Ensure current_seed is a single integer value
|
||||
if isinstance(current_seed, (int, float)):
|
||||
current_seed = int(current_seed)
|
||||
elif isinstance(current_seed, (list, tuple)):
|
||||
current_seed = current_seed[0]
|
||||
else:
|
||||
current_seed = current_seed.iloc[0]
|
||||
|
||||
g1 = g.glitch_image(p, glitch_amount, color_offset=color_offset, seed=current_seed)
|
||||
|
||||
r1 = g1.rotate(90, expand=True)
|
||||
|
||||
g2 = g.glitch_image(r1, glitch_amount, color_offset=color_offset, seed=current_seed)
|
||||
|
||||
f = g2.rotate(-90, expand=True)
|
||||
|
||||
o = np.array(f.convert("RGB")).astype(np.float32) / 255.0
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
# Print progress update
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
@@ -1,68 +1,69 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
class FL_HalftonePattern:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"dot_size": ("INT", {"default": 5, "min": 1, "max": 20, "step": 1}),
|
||||
"dot_spacing": ("INT", {"default": 10, "min": 5, "max": 50, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "halftone_pattern"
|
||||
CATEGORY = "🏵️Fill Nodes/VFX"
|
||||
|
||||
def halftone_pattern(self, images, dot_size=5, dot_spacing=10):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
pbar = ProgressBar(total_images)
|
||||
for i, img in enumerate(images, start=1):
|
||||
img_np = img.cpu().numpy().squeeze()
|
||||
grayscale_image = np.dot(img_np[..., :3], [0.299, 0.587, 0.114])
|
||||
height, width = grayscale_image.shape
|
||||
|
||||
halftone_image = np.ones((height, width), dtype=np.float32)
|
||||
|
||||
for y in range(0, height, dot_spacing):
|
||||
for x in range(0, width, dot_spacing):
|
||||
box = (x, y, x + dot_spacing, y + dot_spacing)
|
||||
region_mean = np.mean(grayscale_image[box[1]:box[3], box[0]:box[2]])
|
||||
dot_radius = int((1 - region_mean) * dot_size / 2)
|
||||
dot_position = (x + dot_spacing // 2, y + dot_spacing // 2)
|
||||
|
||||
# Create a circular mask for the dot
|
||||
y_grid, x_grid = np.ogrid[-dot_radius:dot_radius + 1, -dot_radius:dot_radius + 1]
|
||||
mask = x_grid ** 2 + y_grid ** 2 <= dot_radius ** 2
|
||||
|
||||
# Apply the dot mask to the halftone image
|
||||
y_start = max(0, dot_position[1] - dot_radius)
|
||||
y_end = min(height, dot_position[1] + dot_radius + 1)
|
||||
x_start = max(0, dot_position[0] - dot_radius)
|
||||
x_end = min(width, dot_position[0] + dot_radius + 1)
|
||||
|
||||
# Ensure the mask dimensions match the sliced halftone image dimensions
|
||||
mask_height = y_end - y_start
|
||||
mask_width = x_end - x_start
|
||||
mask = mask[:mask_height, :mask_width]
|
||||
|
||||
try:
|
||||
halftone_image[y_start:y_end, x_start:x_end][mask] = 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
o = np.stack((halftone_image,) * 3, axis=-1)
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
pbar.update_absolute(i)
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
import torch
|
||||
import numpy as np
|
||||
import sys
|
||||
|
||||
class FL_HalftonePattern:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"dot_size": ("INT", {"default": 5, "min": 1, "max": 20, "step": 1}),
|
||||
"dot_spacing": ("INT", {"default": 10, "min": 5, "max": 50, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "halftone_pattern"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def halftone_pattern(self, images, dot_size=5, dot_spacing=10):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
for i, img in enumerate(images, start=1):
|
||||
img_np = img.cpu().numpy().squeeze()
|
||||
grayscale_image = np.dot(img_np[..., :3], [0.299, 0.587, 0.114])
|
||||
height, width = grayscale_image.shape
|
||||
|
||||
halftone_image = np.ones((height, width), dtype=np.float32)
|
||||
|
||||
for y in range(0, height, dot_spacing):
|
||||
for x in range(0, width, dot_spacing):
|
||||
box = (x, y, x + dot_spacing, y + dot_spacing)
|
||||
region_mean = np.mean(grayscale_image[box[1]:box[3], box[0]:box[2]])
|
||||
dot_radius = int((1 - region_mean) * dot_size / 2)
|
||||
dot_position = (x + dot_spacing // 2, y + dot_spacing // 2)
|
||||
|
||||
# Create a circular mask for the dot
|
||||
y_grid, x_grid = np.ogrid[-dot_radius:dot_radius + 1, -dot_radius:dot_radius + 1]
|
||||
mask = x_grid ** 2 + y_grid ** 2 <= dot_radius ** 2
|
||||
|
||||
# Apply the dot mask to the halftone image
|
||||
y_start = max(0, dot_position[1] - dot_radius)
|
||||
y_end = min(height, dot_position[1] + dot_radius + 1)
|
||||
x_start = max(0, dot_position[0] - dot_radius)
|
||||
x_end = min(width, dot_position[0] + dot_radius + 1)
|
||||
|
||||
# Ensure the mask dimensions match the sliced halftone image dimensions
|
||||
mask_height = y_end - y_start
|
||||
mask_width = x_end - x_start
|
||||
mask = mask[:mask_height, :mask_width]
|
||||
|
||||
halftone_image[y_start:y_end, x_start:x_end][mask] = 0
|
||||
|
||||
o = np.stack((halftone_image,) * 3, axis=-1)
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
# Print progress update
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
@@ -1,106 +1,110 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
import math
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
class FL_HexagonalPattern:
|
||||
def __init__(self):
|
||||
self.hexagon_size_index = 0
|
||||
self.shadow_offset_index = 0
|
||||
self.shadow_color_index = 0
|
||||
self.background_color_index = 0
|
||||
self.rotation_index = 0
|
||||
self.spacing_index = 0
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"hexagon_size": ("INT", {"default": 100, "min": 50, "max": 500, "step": 10}),
|
||||
"shadow_offset": ("INT", {"default": 5, "min": 0, "max": 20, "step": 1}),
|
||||
"shadow_color": ("STRING", {"default": "purple"}),
|
||||
"background_color": ("STRING", {"default": "black"}),
|
||||
"rotation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}),
|
||||
"spacing": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "hexagonal_pattern"
|
||||
CATEGORY = "🏵️Fill Nodes/VFX"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def create_hexagon_mask(self, size):
|
||||
mask = Image.new("L", (size, size), 0)
|
||||
draw = ImageDraw.Draw(mask)
|
||||
draw.regular_polygon((size // 2, size // 2, size // 2), 6, fill=255)
|
||||
return mask
|
||||
|
||||
def process_input_value(self, value, index):
|
||||
if isinstance(value, list):
|
||||
if index >= len(value):
|
||||
print(f"Warning: Value list index out of range. Using the last value.")
|
||||
index = len(value) - 1
|
||||
current_value = value[index]
|
||||
index = (index + 1) % len(value)
|
||||
else:
|
||||
current_value = value
|
||||
|
||||
if hasattr(current_value, 'values'):
|
||||
current_value = float(current_value.values[0])
|
||||
|
||||
return current_value, index
|
||||
|
||||
def hexagonal_pattern(self, images, hexagon_size=100, shadow_offset=5, shadow_color="black", shadow_opacity=0.5,
|
||||
background_color="white", rotation=0.0, spacing=1.0):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
pbar = ProgressBar(total_images)
|
||||
for i, img_tensor in enumerate(images, start=1):
|
||||
p = self.t2p(img_tensor)
|
||||
width, height = p.size
|
||||
|
||||
current_hexagon_size, self.hexagon_size_index = self.process_input_value(hexagon_size, self.hexagon_size_index)
|
||||
current_shadow_offset, self.shadow_offset_index = self.process_input_value(shadow_offset, self.shadow_offset_index)
|
||||
current_shadow_color, self.shadow_color_index = self.process_input_value(shadow_color, self.shadow_color_index)
|
||||
current_background_color, self.background_color_index = self.process_input_value(background_color, self.background_color_index)
|
||||
current_rotation, self.rotation_index = self.process_input_value(rotation, self.rotation_index)
|
||||
current_spacing, self.spacing_index = self.process_input_value(spacing, self.spacing_index)
|
||||
|
||||
hexagon_mask = self.create_hexagon_mask(current_hexagon_size)
|
||||
|
||||
output_image = Image.new("RGBA", (width, height), current_background_color)
|
||||
|
||||
for y in range(0, height, int(current_hexagon_size * current_spacing * math.sqrt(3) / 2)):
|
||||
for x in range(0, width, int(current_hexagon_size * current_spacing)):
|
||||
if y % (2 * int(current_hexagon_size * current_spacing * math.sqrt(3) / 2)) == int(current_hexagon_size * current_spacing * math.sqrt(3) / 2):
|
||||
x += int(current_hexagon_size * current_spacing) // 2
|
||||
|
||||
cropped_hexagon = p.crop((x, y, x + current_hexagon_size, y + current_hexagon_size)).rotate(current_rotation, expand=True)
|
||||
|
||||
shadow = Image.new("RGBA", cropped_hexagon.size, (0, 0, 0, 0))
|
||||
shadow_mask = hexagon_mask.copy().resize(cropped_hexagon.size)
|
||||
shadow.paste(current_shadow_color, (current_shadow_offset, current_shadow_offset), shadow_mask)
|
||||
shadow.putalpha(int(255 * shadow_opacity))
|
||||
|
||||
output_image.paste(shadow, (x + current_shadow_offset, y + current_shadow_offset), shadow_mask)
|
||||
output_image.paste(cropped_hexagon, (x, y), shadow_mask)
|
||||
|
||||
o = np.array(output_image.convert("RGB")).astype(np.float32) / 255.0
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
pbar.update_absolute(i)
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
import math
|
||||
import sys
|
||||
|
||||
class FL_HexagonalPattern:
|
||||
def __init__(self):
|
||||
self.hexagon_size_index = 0
|
||||
self.shadow_offset_index = 0
|
||||
self.shadow_color_index = 0
|
||||
self.background_color_index = 0
|
||||
self.rotation_index = 0
|
||||
self.spacing_index = 0
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"hexagon_size": ("INT", {"default": 100, "min": 50, "max": 500, "step": 10}),
|
||||
"shadow_offset": ("INT", {"default": 5, "min": 0, "max": 20, "step": 1}),
|
||||
"shadow_color": ("STRING", {"default": "purple"}),
|
||||
"background_color": ("STRING", {"default": "black"}),
|
||||
"rotation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}),
|
||||
"spacing": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "hexagonal_pattern"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def create_hexagon_mask(self, size):
|
||||
mask = Image.new("L", (size, size), 0)
|
||||
draw = ImageDraw.Draw(mask)
|
||||
draw.regular_polygon((size // 2, size // 2, size // 2), 6, fill=255)
|
||||
return mask
|
||||
|
||||
def process_input_value(self, value, index):
|
||||
if isinstance(value, list):
|
||||
if index >= len(value):
|
||||
print(f"Warning: Value list index out of range. Using the last value.")
|
||||
index = len(value) - 1
|
||||
current_value = value[index]
|
||||
index = (index + 1) % len(value)
|
||||
else:
|
||||
current_value = value
|
||||
|
||||
if hasattr(current_value, 'values'):
|
||||
current_value = float(current_value.values[0])
|
||||
|
||||
return current_value, index
|
||||
|
||||
def hexagonal_pattern(self, images, hexagon_size=100, shadow_offset=5, shadow_color="black", shadow_opacity=0.5,
|
||||
background_color="white", rotation=0.0, spacing=1.0):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
for i, img_tensor in enumerate(images, start=1):
|
||||
p = self.t2p(img_tensor)
|
||||
width, height = p.size
|
||||
|
||||
current_hexagon_size, self.hexagon_size_index = self.process_input_value(hexagon_size, self.hexagon_size_index)
|
||||
current_shadow_offset, self.shadow_offset_index = self.process_input_value(shadow_offset, self.shadow_offset_index)
|
||||
current_shadow_color, self.shadow_color_index = self.process_input_value(shadow_color, self.shadow_color_index)
|
||||
current_background_color, self.background_color_index = self.process_input_value(background_color, self.background_color_index)
|
||||
current_rotation, self.rotation_index = self.process_input_value(rotation, self.rotation_index)
|
||||
current_spacing, self.spacing_index = self.process_input_value(spacing, self.spacing_index)
|
||||
|
||||
hexagon_mask = self.create_hexagon_mask(current_hexagon_size)
|
||||
|
||||
output_image = Image.new("RGBA", (width, height), current_background_color)
|
||||
|
||||
for y in range(0, height, int(current_hexagon_size * current_spacing * math.sqrt(3) / 2)):
|
||||
for x in range(0, width, int(current_hexagon_size * current_spacing)):
|
||||
if y % (2 * int(current_hexagon_size * current_spacing * math.sqrt(3) / 2)) == int(current_hexagon_size * current_spacing * math.sqrt(3) / 2):
|
||||
x += int(current_hexagon_size * current_spacing) // 2
|
||||
|
||||
cropped_hexagon = p.crop((x, y, x + current_hexagon_size, y + current_hexagon_size)).rotate(current_rotation, expand=True)
|
||||
|
||||
shadow = Image.new("RGBA", cropped_hexagon.size, (0, 0, 0, 0))
|
||||
shadow_mask = hexagon_mask.copy().resize(cropped_hexagon.size)
|
||||
shadow.paste(current_shadow_color, (current_shadow_offset, current_shadow_offset), shadow_mask)
|
||||
shadow.putalpha(int(255 * shadow_opacity))
|
||||
|
||||
output_image.paste(shadow, (x + current_shadow_offset, y + current_shadow_offset), shadow_mask)
|
||||
output_image.paste(cropped_hexagon, (x, y), shadow_mask)
|
||||
|
||||
o = np.array(output_image.convert("RGB")).astype(np.float32) / 255.0
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
# Print progress update
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
@@ -0,0 +1,51 @@
|
||||
import os
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class FL_ImageCaptionSaver:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE", {}),
|
||||
"folder_name": ("STRING", {"default": "output_folder"}),
|
||||
"caption_text": ("STRING", {"default": "Your caption here"}),
|
||||
"overwrite": ("BOOLEAN", {"default": True}) # New overwrite toggle
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "save_images_with_captions"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def save_images_with_captions(self, images, folder_name, caption_text, overwrite):
|
||||
# Ensure output directory exists
|
||||
os.makedirs(folder_name, exist_ok=True)
|
||||
|
||||
saved_files = []
|
||||
for i, image_tensor in enumerate(images):
|
||||
base_name = f"image_{i}"
|
||||
image_file_name = f"{folder_name}/{base_name}.png"
|
||||
text_file_name = f"{folder_name}/{base_name}.txt"
|
||||
|
||||
# Check if overwrite is disabled and file exists
|
||||
if not overwrite:
|
||||
counter = 1
|
||||
while os.path.exists(image_file_name) or os.path.exists(text_file_name):
|
||||
image_file_name = f"{folder_name}/{base_name}_{counter}.png"
|
||||
text_file_name = f"{folder_name}/{base_name}_{counter}.txt"
|
||||
counter += 1
|
||||
|
||||
# Convert tensor to image
|
||||
image = Image.fromarray((image_tensor.numpy() * 255).astype('uint8'), 'RGB')
|
||||
|
||||
# Save image
|
||||
image.save(image_file_name)
|
||||
saved_files.append(image_file_name)
|
||||
|
||||
# Save text file
|
||||
with open(text_file_name, "w") as text_file:
|
||||
text_file.write(caption_text)
|
||||
|
||||
return (f"Saved {len(images)} images and captions in '{folder_name}'",)
|
||||
@@ -1,35 +1,39 @@
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
class FL_ImageDimensionDisplay:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE", {}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "display_dimensions"
|
||||
CATEGORY = "🏵️Fill Nodes/Image"
|
||||
|
||||
def display_dimensions(self, image):
|
||||
if isinstance(image, torch.Tensor):
|
||||
if image.dim() == 4: # Batch dimension is present
|
||||
_, height, width, _ = image.shape
|
||||
elif image.dim() == 3: # No batch dimension, single image
|
||||
height, width, _ = image.shape
|
||||
else:
|
||||
return ("Unsupported tensor format",)
|
||||
elif isinstance(image, Image.Image):
|
||||
width, height = image.size
|
||||
else:
|
||||
return ("Unsupported image format",)
|
||||
|
||||
dimensions = f"Width: {width}, Height: {height}"
|
||||
return (dimensions,)
|
||||
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, image):
|
||||
return float("NaN") # This ensures the node always updates
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class FL_ImageDimensionDisplay:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE", {}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "display_dimensions"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def display_dimensions(self, image):
|
||||
# Check the number of dimensions in the image tensor to correctly unpack the dimensions
|
||||
if isinstance(image, torch.Tensor):
|
||||
if image.dim() == 4: # Batch dimension is present
|
||||
_, height, width, _, = image.shape
|
||||
elif image.dim() == 3: # No batch dimension, single image
|
||||
_, height, width = image.shape
|
||||
else:
|
||||
return ("Unsupported tensor format",)
|
||||
elif isinstance(image, Image.Image):
|
||||
width, height = image.size
|
||||
else:
|
||||
return ("Unsupported image format",)
|
||||
|
||||
# Correctly assign width and height
|
||||
dimensions = f"Width: {width}, Height: {height}"
|
||||
|
||||
# Display dimensions in the UI. This might need to be adapted.
|
||||
print(dimensions)
|
||||
|
||||
# Return the dimensions as a string.
|
||||
return (dimensions,)
|
||||
@@ -0,0 +1,36 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
class FL_ImageDurationSync:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE", {}),
|
||||
"frame_count": ("INT", {"default": 30}),
|
||||
"bpm": ("INT", {"default": 120}),
|
||||
"fps": ("INT", {"default": 30}),
|
||||
"bars": ("FLOAT", {"default": 4.0, "step": 0.05}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "INT")
|
||||
RETURN_NAMES = ("output_images", "hold_frames")
|
||||
FUNCTION = "sync_image_to_duration"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def sync_image_to_duration(self, images, bpm, frame_count, bars, fps):
|
||||
# Calculate the duration of each bar in seconds
|
||||
bar_duration = 60 / bpm * 4
|
||||
|
||||
# Calculate the total duration in seconds
|
||||
total_duration = bar_duration * bars
|
||||
|
||||
# Calculate the number of frames to hold the image
|
||||
hold_frames = int(total_duration * fps)
|
||||
|
||||
# Repeat the image for the calculated number of frames
|
||||
output_images = images.repeat(hold_frames, 1, 1, 1)
|
||||
|
||||
return (output_images, hold_frames)
|
||||
@@ -0,0 +1,89 @@
|
||||
import torch
|
||||
from PIL import Image
|
||||
from kornia.morphology import gradient
|
||||
import comfy.model_management
|
||||
import sys
|
||||
|
||||
class FL_ImagePixelator:
|
||||
def __init__(self):
|
||||
self.modulation_index = 0
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE", {}),
|
||||
"scale_factor": ("FLOAT", {"default": 0.0500, "min": 0.0100, "max": 0.2000, "step": 0.0100}),
|
||||
"kernel_size": ("INT", {"default": 3, "max": 10, "step": 1}),
|
||||
"modulation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "pixelate_image"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def pixelate_image(self, image, scale_factor, kernel_size, modulation):
|
||||
if isinstance(image, torch.Tensor):
|
||||
if image.dim() == 4: # Batch dimension is present
|
||||
output_images = []
|
||||
total_frames = image.shape[0]
|
||||
for i, single_image in enumerate(image, start=1):
|
||||
single_image = single_image.unsqueeze(0) # Add batch dimension
|
||||
modulated_scale_factor = self.apply_modulation(scale_factor, modulation, total_frames)
|
||||
single_image = self.apply_pixelation_tensor(single_image, modulated_scale_factor)
|
||||
single_image = self.process(single_image, kernel_size)
|
||||
output_images.append(single_image)
|
||||
progress = i / total_frames * 100
|
||||
sys.stdout.write(f"\rProcessing frames: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
image = torch.cat(output_images, dim=0) # Concatenate processed images along batch dimension
|
||||
elif image.dim() == 3: # No batch dimension, single image
|
||||
image = image.unsqueeze(0) # Add batch dimension
|
||||
modulated_scale_factor = self.apply_modulation(scale_factor, modulation, 1)
|
||||
image = self.apply_pixelation_tensor(image, modulated_scale_factor)
|
||||
image = self.process(image, kernel_size)
|
||||
image = image.squeeze(0) # Remove batch dimension
|
||||
print("Processing single image")
|
||||
else:
|
||||
return (None,)
|
||||
elif isinstance(image, Image.Image):
|
||||
modulated_scale_factor = self.apply_modulation(scale_factor, modulation, 1)
|
||||
image = self.apply_pixelation_pil(image, modulated_scale_factor)
|
||||
image = self.process(image, kernel_size)
|
||||
print("Processing single PIL image")
|
||||
else:
|
||||
return (None,)
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
return (image,)
|
||||
|
||||
def apply_modulation(self, scale_factor, modulation, total_frames):
|
||||
modulation_factor = 1 + modulation * torch.sin(2 * torch.pi * torch.tensor(self.modulation_index / total_frames))
|
||||
modulated_scale_factor = scale_factor * modulation_factor.item()
|
||||
self.modulation_index += 1
|
||||
return modulated_scale_factor
|
||||
|
||||
def apply_pixelation_pil(self, input_image, scale_factor):
|
||||
width, height = input_image.size
|
||||
new_size = (int(width * scale_factor), int(height * scale_factor))
|
||||
resized_image = input_image.resize(new_size, Image.NEAREST)
|
||||
pixelated_image = resized_image.resize((width, height), Image.NEAREST)
|
||||
return pixelated_image
|
||||
|
||||
def apply_pixelation_tensor(self, input_image, scale_factor):
|
||||
_, num_channels, height, width = input_image.shape
|
||||
new_height, new_width = max(1, int(height * scale_factor)), max(1, int(width * scale_factor))
|
||||
resized_tensor = torch.nn.functional.interpolate(input_image, size=(new_height, new_width), mode='nearest')
|
||||
output_tensor = torch.nn.functional.interpolate(resized_tensor, size=(height, width), mode='nearest')
|
||||
return output_tensor
|
||||
|
||||
def process(self, image, kernel_size):
|
||||
device = comfy.model_management.get_torch_device()
|
||||
kernel = torch.ones(kernel_size, kernel_size, device=device)
|
||||
image_k = image.to(device).movedim(-1, 1)
|
||||
output = gradient(image_k, kernel)
|
||||
img_out = output.to(comfy.model_management.intermediate_device()).movedim(1, -1)
|
||||
return img_out
|
||||
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import random
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
class FL_ImageRandomizer:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"directory_path": ("STRING", {"default": ""}),
|
||||
"randomize": ("BOOLEAN", {"default": True}), # Toggle for randomization
|
||||
"run_trigger": ("INT", {"default": 0}), # Dummy input for caching issue
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "PATH") # Adjusted to include image path for preview
|
||||
FUNCTION = "select_image"
|
||||
CATEGORY = "🏵️Fill Nodes" # Adjusted to appear under "Fill Nodes"
|
||||
|
||||
def __init__(self):
|
||||
self.last_index = -1
|
||||
|
||||
def select_image(self, directory_path, randomize, run_trigger):
|
||||
if not directory_path:
|
||||
raise ValueError("Directory path is not provided.")
|
||||
|
||||
images = self.load_images(directory_path)
|
||||
if not images:
|
||||
raise ValueError("No images found in the specified directory.")
|
||||
|
||||
if randomize:
|
||||
selected_image_path = random.choice(images)
|
||||
else:
|
||||
self.last_index = (self.last_index + 1) % len(images)
|
||||
selected_image_path = images[self.last_index]
|
||||
|
||||
image = Image.open(selected_image_path)
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image = image.convert("RGB")
|
||||
image_np = np.array(image).astype(np.float32) / 255.0
|
||||
image_tensor = torch.from_numpy(image_np)[None,]
|
||||
|
||||
return (image_tensor, selected_image_path) # Return both data points
|
||||
|
||||
def load_images(self, directory):
|
||||
supported_formats = ["jpg", "jpeg", "png", "bmp", "gif"]
|
||||
return [os.path.join(directory, f) for f in os.listdir(directory)
|
||||
if os.path.isfile(os.path.join(directory, f)) and f.split('.')[-1].lower() in supported_formats]
|
||||
@@ -0,0 +1,226 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import sys
|
||||
import OpenGL.GL as gl
|
||||
import glfw
|
||||
import ctypes
|
||||
|
||||
VERTEX_SHADER = """
|
||||
#version 330 core
|
||||
|
||||
layout (location = 0) in vec3 aPos;
|
||||
layout (location = 1) in vec2 aTexCoord;
|
||||
|
||||
out vec2 TexCoord;
|
||||
|
||||
void main()
|
||||
{
|
||||
gl_Position = vec4(aPos, 1.0);
|
||||
TexCoord = aTexCoord;
|
||||
}
|
||||
"""
|
||||
|
||||
FRAGMENT_SHADER = """
|
||||
#version 330 core
|
||||
|
||||
out vec4 FragColor;
|
||||
|
||||
in vec2 TexCoord;
|
||||
|
||||
uniform sampler2D iChannel0;
|
||||
uniform vec3 iResolution;
|
||||
uniform float iTime;
|
||||
uniform float iScale;
|
||||
uniform float iSwirl;
|
||||
uniform float iSwirlStrength;
|
||||
uniform float iBlur;
|
||||
uniform float iIterations;
|
||||
uniform float iTimeSpeed;
|
||||
|
||||
void main()
|
||||
{
|
||||
vec4 O = vec4(0.0);
|
||||
vec2 U = TexCoord;
|
||||
|
||||
float s = 0.0, s2 = 0.0, t = iTime * iTimeSpeed;
|
||||
U = U - 0.5;
|
||||
U.x += 0.03 * sin(1.14 * t);
|
||||
float sc = pow(iScale, -mod(t, 2.0) - 0.8);
|
||||
U *= sc;
|
||||
|
||||
for (int i = 0; i < int(iIterations); i++) {
|
||||
vec2 V = abs(U + U);
|
||||
if (max(V.x, V.y) > 1.0) break;
|
||||
V = smoothstep(1.0, 0.5, V);
|
||||
float m = V.x * V.y;
|
||||
O = mix(O, texture(iChannel0, U + 0.5), m);
|
||||
s = mix(s, 1.0, m);
|
||||
s2 = s2 * (1.0 - m) * (1.0 - m) + m * m;
|
||||
U *= iScale;
|
||||
if (iSwirl > 0.5) {
|
||||
U.x = -U.x * iSwirlStrength;
|
||||
}
|
||||
}
|
||||
|
||||
vec4 mean = texture(iChannel0, U, 10.0);
|
||||
O = mean + (O - s * mean) / sqrt(s2);
|
||||
|
||||
FragColor = O;
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
class FL_InfiniteZoom:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"scale": ("FLOAT", {"default": 2.00, "min": 1.10, "max": 10.00, "step": 0.05}),
|
||||
"mirror": (["on", "off"],),
|
||||
"mirror_warp": ("FLOAT", {"default": 1.00, "min": 0.50, "max": 1.50, "step": 0.05}),
|
||||
"iterations": ("INT", {"default": 10, "min": 1, "max": 100, "step": 1}),
|
||||
#"blur": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}),
|
||||
"speed": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}),
|
||||
"fps": ("INT", {"default": 30, "min": 1, "max": 120, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "apply_shader"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def apply_shader(self, images, scale, mirror, mirror_warp, iterations, speed, fps):
|
||||
result = []
|
||||
total_images = len(images)
|
||||
frame_time = 1.0 / fps
|
||||
|
||||
for i, image in enumerate(images, start=1):
|
||||
img = self.t2p(image)
|
||||
result_img = self.process_image(img, scale, mirror, mirror_warp, iterations, speed,
|
||||
i * frame_time)
|
||||
result_img = self.p2t(result_img)
|
||||
result.append(result_img)
|
||||
|
||||
# Update the print log
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress log
|
||||
print()
|
||||
|
||||
return (torch.cat(result, dim=0),)
|
||||
|
||||
def process_image(self, image, scale, mirror, mirror_warp, iterations, speed, time):
|
||||
# Convert the PIL image to a numpy array
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
|
||||
# Create a PyOpenGL context
|
||||
if not glfw.init():
|
||||
raise RuntimeError("Failed to initialize GLFW")
|
||||
|
||||
glfw.window_hint(glfw.VISIBLE, glfw.FALSE)
|
||||
window = glfw.create_window(image.width, image.height, "Hidden Window", None, None)
|
||||
if not window:
|
||||
glfw.terminate()
|
||||
raise RuntimeError("Failed to create GLFW window")
|
||||
|
||||
glfw.make_context_current(window)
|
||||
|
||||
# Compile the shader program
|
||||
vertex_shader = gl.glCreateShader(gl.GL_VERTEX_SHADER)
|
||||
gl.glShaderSource(vertex_shader, VERTEX_SHADER)
|
||||
gl.glCompileShader(vertex_shader)
|
||||
|
||||
fragment_shader = gl.glCreateShader(gl.GL_FRAGMENT_SHADER)
|
||||
gl.glShaderSource(fragment_shader, FRAGMENT_SHADER)
|
||||
gl.glCompileShader(fragment_shader)
|
||||
|
||||
shader_program = gl.glCreateProgram()
|
||||
gl.glAttachShader(shader_program, vertex_shader)
|
||||
gl.glAttachShader(shader_program, fragment_shader)
|
||||
gl.glLinkProgram(shader_program)
|
||||
|
||||
gl.glUseProgram(shader_program)
|
||||
|
||||
# Set up vertex buffer object (VBO) and vertex array object (VAO)
|
||||
vertices = np.array([
|
||||
-1.0, -1.0, 0.0, 0.0, 0.0,
|
||||
1.0, -1.0, 0.0, 1.0, 0.0,
|
||||
-1.0, 1.0, 0.0, 0.0, 1.0,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0
|
||||
], dtype=np.float32)
|
||||
|
||||
vao = gl.glGenVertexArrays(1)
|
||||
gl.glBindVertexArray(vao)
|
||||
|
||||
vbo = gl.glGenBuffers(1)
|
||||
gl.glBindBuffer(gl.GL_ARRAY_BUFFER, vbo)
|
||||
gl.glBufferData(gl.GL_ARRAY_BUFFER, vertices.nbytes, vertices, gl.GL_STATIC_DRAW)
|
||||
|
||||
gl.glVertexAttribPointer(0, 3, gl.GL_FLOAT, gl.GL_FALSE, 5 * vertices.itemsize, None)
|
||||
gl.glEnableVertexAttribArray(0)
|
||||
|
||||
gl.glVertexAttribPointer(1, 2, gl.GL_FLOAT, gl.GL_FALSE, 5 * vertices.itemsize,
|
||||
ctypes.c_void_p(3 * vertices.itemsize))
|
||||
gl.glEnableVertexAttribArray(1)
|
||||
|
||||
# Set up texture
|
||||
texture = gl.glGenTextures(1)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.width, image.height, 0, gl.GL_RGB, gl.GL_FLOAT, img_array)
|
||||
|
||||
# Set shader uniforms
|
||||
gl.glUniform1i(gl.glGetUniformLocation(shader_program, "iChannel0"), 0)
|
||||
gl.glUniform3f(gl.glGetUniformLocation(shader_program, "iResolution"), image.width, image.height, 0.0)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iTime"), time)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iScale"), scale)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iSwirl"), 1.0 if mirror == "on" else 0.0)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iSwirlStrength"), mirror_warp)
|
||||
# gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iBlur"), blur)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iIterations"), iterations)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iTimeSpeed"), speed)
|
||||
|
||||
# Render the shader
|
||||
gl.glClear(gl.GL_COLOR_BUFFER_BIT)
|
||||
gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4)
|
||||
|
||||
# Read the rendered image from the framebuffer
|
||||
img_data = gl.glReadPixels(0, 0, image.width, image.height, gl.GL_RGB, gl.GL_FLOAT)
|
||||
img_array = np.frombuffer(img_data, dtype=np.float32).reshape((image.height, image.width, 3))
|
||||
|
||||
# Clean up OpenGL resources
|
||||
gl.glDeleteTextures(1, [texture])
|
||||
gl.glDeleteBuffers(1, [vbo])
|
||||
gl.glDeleteVertexArrays(1, [vao])
|
||||
gl.glDeleteProgram(shader_program)
|
||||
gl.glDeleteShader(vertex_shader)
|
||||
gl.glDeleteShader(fragment_shader)
|
||||
|
||||
glfw.destroy_window(window)
|
||||
glfw.terminate()
|
||||
|
||||
# Convert the processed image back to a PIL image
|
||||
processed_image = Image.fromarray((img_array * 255).astype(np.uint8))
|
||||
|
||||
return processed_image
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def p2t(self, p):
|
||||
if p is not None:
|
||||
i = np.array(p).astype(np.float32) / 255.0
|
||||
t = torch.from_numpy(i).unsqueeze(0)
|
||||
return t
|
||||
@@ -1,90 +1,85 @@
|
||||
import fnmatch
|
||||
import os
|
||||
import torch
|
||||
import random
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from ..sup import ROOT
|
||||
|
||||
class FL_NFTGenerator:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"folder_path": ("STRING", {"default": ""}),
|
||||
"dummy_seed": ("INT", {"default": 0, "min": 0, "max": 1000000}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "IMAGE")
|
||||
FUNCTION = "generate_nft"
|
||||
CATEGORY = "🏵️Fill Nodes/experiments"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def generate_nft(self, folder_path, dummy_seed):
|
||||
if not (path := Path(ROOT / folder_path)).is_dir():
|
||||
if not (path := Path(folder_path)).is_dir():
|
||||
raise ValueError(f"Folder path does not exist: {folder_path}")
|
||||
|
||||
image_files = [str(f) for f in path.glob('*') if not fnmatch.fnmatch(f.name, '*-mask.*')]
|
||||
if len(image_files) == 0:
|
||||
raise ValueError(f"No image files found in the folder: {folder_path}")
|
||||
|
||||
# Extract rarity percentages from image filenames
|
||||
rarities = []
|
||||
for image_file in image_files:
|
||||
if "-" in image_file:
|
||||
# name-alexperval-was
|
||||
rarity_str = image_file.split("-")[1].split("per")[0]
|
||||
rarity = int(rarity_str)
|
||||
rarities.append(rarity)
|
||||
else:
|
||||
raise ValueError(f"Invalid image filename format: {image_file}")
|
||||
|
||||
# Calculate cumulative probabilities
|
||||
total_rarity = sum(rarities)
|
||||
probabilities = [rarity / total_rarity for rarity in rarities]
|
||||
cumulative_probabilities = [sum(probabilities[:i+1]) for i in range(len(probabilities))]
|
||||
|
||||
# Generate a random number between 0 and 1 using the dummy seed
|
||||
random.seed(dummy_seed)
|
||||
random_number = random.random()
|
||||
|
||||
# Find the index of the selected image based on the random number and cumulative probabilities
|
||||
selected_index = None
|
||||
for i, prob in enumerate(cumulative_probabilities):
|
||||
if random_number <= prob:
|
||||
selected_index = i
|
||||
break
|
||||
|
||||
if selected_index is None:
|
||||
raise ValueError("Failed to select an image based on rarity.")
|
||||
|
||||
# Get the selected image and its corresponding mask
|
||||
selected_image_file = image_files[selected_index]
|
||||
# selected_image_path = os.path.join(folder_path, selected_image_file)
|
||||
selected_image = Image.open(selected_image_file)
|
||||
|
||||
# Get the file extension of the selected image
|
||||
_, extension = os.path.splitext(selected_image_file)
|
||||
|
||||
# Generate the mask filename based on the selected image filename
|
||||
mask_file = selected_image_file.rsplit(".", 1)[0] + "-mask" + extension
|
||||
mask_path = os.path.join(folder_path, mask_file)
|
||||
|
||||
if os.path.exists(mask_path):
|
||||
mask_image = Image.open(mask_path)
|
||||
else:
|
||||
# Create a blank mask image if the corresponding mask is not found
|
||||
mask_image = Image.new("RGB", selected_image.size, (0, 0, 0))
|
||||
|
||||
selected_image_tensor = torch.from_numpy(np.array(selected_image).astype(np.float32) / 255.0).unsqueeze(0)
|
||||
mask_image_tensor = torch.from_numpy(np.array(mask_image).astype(np.float32) / 255.0).unsqueeze(0)
|
||||
|
||||
import os
|
||||
import random
|
||||
from PIL import Image
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
class FL_NFTGenerator:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"folder_path": ("STRING", {"default": ""}),
|
||||
"dummy_seed": ("INT", {"default": 0, "min": 0, "max": 1000000}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "IMAGE")
|
||||
FUNCTION = "generate_nft"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def generate_nft(self, folder_path, dummy_seed):
|
||||
if not os.path.exists(folder_path):
|
||||
raise ValueError(f"Folder path does not exist: {folder_path}")
|
||||
|
||||
image_files = [f for f in os.listdir(folder_path) if not f.lower().endswith("-mask.png") and not f.lower().endswith("-mask.jpg") and not f.lower().endswith("-mask.jpeg")]
|
||||
if not image_files:
|
||||
raise ValueError(f"No image files found in the folder: {folder_path}")
|
||||
|
||||
# Extract rarity percentages from image filenames
|
||||
rarities = []
|
||||
for image_file in image_files:
|
||||
if "-" in image_file:
|
||||
rarity_str = image_file.split("-")[1].split("per")[0]
|
||||
rarity = int(rarity_str)
|
||||
rarities.append(rarity)
|
||||
else:
|
||||
raise ValueError(f"Invalid image filename format: {image_file}")
|
||||
|
||||
# Calculate cumulative probabilities
|
||||
total_rarity = sum(rarities)
|
||||
probabilities = [rarity / total_rarity for rarity in rarities]
|
||||
cumulative_probabilities = [sum(probabilities[:i+1]) for i in range(len(probabilities))]
|
||||
|
||||
# Generate a random number between 0 and 1 using the dummy seed
|
||||
random.seed(dummy_seed)
|
||||
random_number = random.random()
|
||||
|
||||
# Find the index of the selected image based on the random number and cumulative probabilities
|
||||
selected_index = None
|
||||
for i, prob in enumerate(cumulative_probabilities):
|
||||
if random_number <= prob:
|
||||
selected_index = i
|
||||
break
|
||||
|
||||
if selected_index is None:
|
||||
raise ValueError("Failed to select an image based on rarity.")
|
||||
|
||||
# Get the selected image and its corresponding mask
|
||||
selected_image_file = image_files[selected_index]
|
||||
selected_image_path = os.path.join(folder_path, selected_image_file)
|
||||
selected_image = Image.open(selected_image_path)
|
||||
|
||||
# Get the file extension of the selected image
|
||||
_, extension = os.path.splitext(selected_image_file)
|
||||
|
||||
# Generate the mask filename based on the selected image filename
|
||||
mask_file = selected_image_file.rsplit(".", 1)[0] + "-mask" + extension
|
||||
mask_path = os.path.join(folder_path, mask_file)
|
||||
|
||||
if os.path.exists(mask_path):
|
||||
mask_image = Image.open(mask_path)
|
||||
else:
|
||||
# Create a blank mask image if the corresponding mask is not found
|
||||
mask_image = Image.new("RGB", selected_image.size, (0, 0, 0))
|
||||
|
||||
selected_image_tensor = torch.from_numpy(np.array(selected_image).astype(np.float32) / 255.0).unsqueeze(0)
|
||||
mask_image_tensor = torch.from_numpy(np.array(mask_image).astype(np.float32) / 255.0).unsqueeze(0)
|
||||
|
||||
return (selected_image_tensor, mask_image_tensor)
|
||||
@@ -1,471 +1,297 @@
|
||||
import glfw
|
||||
import ctypes
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import OpenGL.GL as gl
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
from ..audio.audio_envelope import load_audio_envelope
|
||||
|
||||
VERTEX_SHADER = """
|
||||
#version 330 core
|
||||
|
||||
layout (location = 0) in vec3 aPos;
|
||||
layout (location = 1) in vec2 aTexCoord;
|
||||
|
||||
out vec2 TexCoord;
|
||||
|
||||
void main()
|
||||
{
|
||||
gl_Position = vec4(aPos, 1.0);
|
||||
TexCoord = aTexCoord;
|
||||
}
|
||||
"""
|
||||
|
||||
FRAGMENT_SHADER = """
|
||||
#version 330 core
|
||||
|
||||
out vec4 FragColor;
|
||||
|
||||
in vec2 TexCoord;
|
||||
|
||||
uniform sampler2D iChannel0;
|
||||
uniform sampler2D iChannel1;
|
||||
uniform vec3 iResolution;
|
||||
uniform float iTime;
|
||||
uniform float iAngleNum;
|
||||
uniform float iSampNum;
|
||||
uniform float iLineWidth;
|
||||
uniform float iVignette;
|
||||
|
||||
#define Res0 textureSize(iChannel0, 0)
|
||||
#define Res1 textureSize(iChannel1, 0)
|
||||
#define Res iResolution.xy
|
||||
|
||||
#define randSamp iChannel1
|
||||
#define colorSamp iChannel0
|
||||
|
||||
vec4 getRand(vec2 pos)
|
||||
{
|
||||
return textureLod(iChannel1, pos / Res1 / iResolution.y * 1080., 0.0);
|
||||
}
|
||||
|
||||
vec4 getCol(vec2 pos)
|
||||
{
|
||||
vec2 uv = ((pos - Res.xy * .5) / Res.y * Res0.y) / Res0.xy + .5;
|
||||
vec4 c1 = texture(iChannel0, uv);
|
||||
vec4 e = smoothstep(vec4(-0.05), vec4(-0.0), vec4(uv, vec2(1) - uv));
|
||||
c1 = mix(vec4(1, 1, 1, 0), c1, e.x * e.y * e.z * e.w);
|
||||
float d = clamp(dot(c1.xyz, vec3(-.5, 1., -.5)), 0.0, 1.0);
|
||||
vec4 c2 = vec4(.7);
|
||||
return min(mix(c1, c2, 1.8 * d), .7);
|
||||
}
|
||||
|
||||
vec4 getColHT(vec2 pos)
|
||||
{
|
||||
return smoothstep(.95, 1.05, getCol(pos) * .8 + .2 + getRand(pos * .7));
|
||||
}
|
||||
|
||||
float getVal(vec2 pos)
|
||||
{
|
||||
vec4 c = getCol(pos);
|
||||
return pow(dot(c.xyz, vec3(.333)), 1.) * 1.;
|
||||
}
|
||||
|
||||
vec2 getGrad(vec2 pos, float eps)
|
||||
{
|
||||
vec2 d = vec2(eps, 0);
|
||||
return vec2(
|
||||
getVal(pos + d.xy) - getVal(pos - d.xy),
|
||||
getVal(pos + d.yx) - getVal(pos - d.yx)
|
||||
) / eps / 2.;
|
||||
}
|
||||
|
||||
#define PI2 6.28318530717959
|
||||
|
||||
void main()
|
||||
{
|
||||
vec2 pos = TexCoord * iResolution.xy + 4.0 * sin(iTime * 1. * vec2(1, 1.7)) * iResolution.y / 400.;
|
||||
vec3 col = vec3(0);
|
||||
vec3 col2 = vec3(0);
|
||||
float sum = 0.;
|
||||
for (int i = 0; i < int(iAngleNum); i++)
|
||||
{
|
||||
float ang = PI2 / iAngleNum * (float(i) + .8);
|
||||
vec2 v = vec2(cos(ang), sin(ang));
|
||||
for (int j = 0; j < int(iSampNum); j++)
|
||||
{
|
||||
vec2 dpos = v.yx * vec2(1, -1) * float(j) * iLineWidth * iResolution.y / 400.;
|
||||
vec2 dpos2 = v.xy * float(j * j) / iSampNum * .5 * iLineWidth * iResolution.y / 400.;
|
||||
vec2 g;
|
||||
float fact;
|
||||
float fact2;
|
||||
|
||||
for (float s = -1.; s <= 1.; s += 2.)
|
||||
{
|
||||
vec2 pos2 = pos + s * dpos + dpos2;
|
||||
vec2 pos3 = pos + (s * dpos + dpos2).yx * vec2(1, -1) * 2.;
|
||||
g = getGrad(pos2, .4);
|
||||
fact = dot(g, v) - .5 * abs(dot(g, v.yx * vec2(1, -1)));
|
||||
fact2 = dot(normalize(g + vec2(.0001)), v.yx * vec2(1, -1));
|
||||
|
||||
fact = clamp(fact, 0., .05);
|
||||
fact2 = abs(fact2);
|
||||
|
||||
fact *= 1. - float(j) / iSampNum;
|
||||
col += fact;
|
||||
col2 += fact2 * getColHT(pos3).xyz;
|
||||
sum += fact2;
|
||||
}
|
||||
}
|
||||
}
|
||||
col /= iSampNum * iAngleNum * .75 / sqrt(iResolution.y);
|
||||
col2 /= sum;
|
||||
col.x *= (.6 + .8 * getRand(pos * .7).x);
|
||||
col.x = 1. - col.x;
|
||||
col.x *= col.x * col.x;
|
||||
|
||||
vec2 s = sin(pos.xy * .1 / sqrt(iResolution.y / 400.));
|
||||
vec3 karo = vec3(1);
|
||||
karo -= .5 * vec3(.25, .1, .1) * dot(exp(-s * s * 80.), vec2(1));
|
||||
float r = length(pos - iResolution.xy * .5) / iResolution.x;
|
||||
float vign = 1. - r * r * r * iVignette;
|
||||
FragColor = vec4(vec3(col.x * col2 * karo * vign), 1);
|
||||
}
|
||||
"""
|
||||
|
||||
class FL_PaperDrawn:
|
||||
def t2p(self, t):
|
||||
"""Tensor to PIL"""
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def p2t(self, p):
|
||||
"""PIL to Tensor"""
|
||||
if p is not None:
|
||||
i = np.array(p).astype(np.float32) / 255.0
|
||||
t = torch.from_numpy(i).unsqueeze(0)
|
||||
return t
|
||||
|
||||
def prepare_mask_batch(self, mask, total_images):
|
||||
"""Prepare mask batch to match image batch size"""
|
||||
if mask is None:
|
||||
return None
|
||||
mask_images = [self.t2p(m) for m in mask]
|
||||
if len(mask_images) < total_images:
|
||||
mask_images = mask_images * (total_images // len(mask_images) + 1)
|
||||
return mask_images[:total_images]
|
||||
|
||||
def process_mask(self, mask, target_size):
|
||||
"""Resize and convert mask to grayscale"""
|
||||
mask = mask.resize(target_size, Image.LANCZOS)
|
||||
return mask.convert('L') if mask.mode != 'L' else mask
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
# Audio reactivity (optional - at top for visibility)
|
||||
"envelope": ("FL_AUDIO_ENVELOPE", {"description": "Optional FL audio envelope for reactive blending"}),
|
||||
"blend_intensity": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05, "description": "Audio-reactive blend intensity"}),
|
||||
"invert": ("BOOLEAN", {"default": False, "description": "Invert envelope (show sketch when quiet)"}),
|
||||
"mask": ("IMAGE", {"default": None, "description": "Optional mask to control where effect is applied"}),
|
||||
# Effect parameters
|
||||
"angle_num": ("FLOAT", {"default": 3.0, "min": 1.0, "max": 10.0, "step": 1.0, "description": "Number of sketch angles"}),
|
||||
"samp_num": ("FLOAT", {"default": 2.2, "min": 1.0, "max": 10.0, "step": 0.1, "description": "Sample density"}),
|
||||
"line_width": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1, "description": "Line thickness"}),
|
||||
"vignette": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.1, "description": "Vignette strength"}),
|
||||
"fps": ("INT", {"default": 30, "min": 1, "max": 120, "step": 1, "description": "Frames per second"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "apply_shader"
|
||||
CATEGORY = "🏵️Fill Nodes/VFX"
|
||||
|
||||
def apply_shader(self, image, angle_num=3.0, samp_num=2.2, line_width=1.0, vignette=0.0, fps=30, envelope=None, blend_intensity=1.0, invert=False, mask=None):
|
||||
# Check if audio-reactive mode is enabled
|
||||
use_audio_reactive = envelope is not None
|
||||
|
||||
if use_audio_reactive:
|
||||
return self._apply_audio_reactive(image, angle_num, samp_num, line_width, vignette, fps, envelope, blend_intensity, invert, mask)
|
||||
else:
|
||||
return self._apply_static(image, angle_num, samp_num, line_width, vignette, fps, mask)
|
||||
|
||||
def _apply_static(self, image, angle_num, samp_num, line_width, vignette, fps, mask=None):
|
||||
"""Original static paper drawn effect"""
|
||||
result = []
|
||||
total_images = len(image)
|
||||
frame_time = 1.0 / fps
|
||||
|
||||
# Prepare mask batch if provided
|
||||
mask_images = self.prepare_mask_batch(mask, total_images) if mask is not None else None
|
||||
|
||||
pbar = ProgressBar(total_images)
|
||||
for i, img in enumerate(image, start=1):
|
||||
img_pil = self.t2p(img)
|
||||
result_img = self.process_image(img_pil, angle_num, samp_num, line_width, vignette, i * frame_time)
|
||||
|
||||
# Apply mask if provided
|
||||
if mask_images is not None:
|
||||
mask_img = self.process_mask(mask_images[i-1], result_img.size)
|
||||
# Blend original and effect based on mask
|
||||
result_array = np.array(result_img).astype(np.float32)
|
||||
original_array = np.array(img_pil).astype(np.float32)
|
||||
mask_array = np.array(mask_img).astype(np.float32) / 255.0
|
||||
|
||||
# Expand mask to 3 channels
|
||||
if len(mask_array.shape) == 2:
|
||||
mask_array = np.stack([mask_array] * 3, axis=-1)
|
||||
|
||||
# Blend: mask=1.0 shows effect, mask=0.0 shows original
|
||||
blended = result_array * mask_array + original_array * (1.0 - mask_array)
|
||||
result_img = Image.fromarray(blended.astype(np.uint8))
|
||||
|
||||
result_img = self.p2t(result_img)
|
||||
result.append(result_img)
|
||||
pbar.update_absolute(i)
|
||||
|
||||
return (torch.cat(result, dim=0),)
|
||||
|
||||
def _apply_audio_reactive(self, image, angle_num, samp_num, line_width, vignette, fps, envelope, blend_intensity, invert, mask=None):
|
||||
"""Audio-reactive blending between original and paper drawn effect"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[FL Paper Drawn] Audio-reactive mode enabled")
|
||||
print(f"[FL Paper Drawn] Effect params: angle={angle_num}, samp={samp_num}, line={line_width}, vignette={vignette}")
|
||||
print(f"[FL Paper Drawn] Blend intensity = {blend_intensity}, Invert = {invert}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
try:
|
||||
envelope_values = load_audio_envelope(envelope)["values"]
|
||||
|
||||
batch_size = len(image)
|
||||
num_envelope_frames = len(envelope_values)
|
||||
|
||||
print(f"[FL Paper Drawn] Input frames: {batch_size}")
|
||||
print(f"[FL Paper Drawn] Envelope frames: {num_envelope_frames}")
|
||||
|
||||
# Handle frame count mismatch
|
||||
if batch_size != num_envelope_frames:
|
||||
print(f"[FL Paper Drawn] WARNING: Frame count mismatch! Using min({batch_size}, {num_envelope_frames})")
|
||||
max_frames = min(batch_size, num_envelope_frames)
|
||||
else:
|
||||
max_frames = batch_size
|
||||
|
||||
# Prepare mask batch if provided
|
||||
mask_images = self.prepare_mask_batch(mask, max_frames) if mask is not None else None
|
||||
|
||||
# First pass: Generate paper drawn effect for all frames
|
||||
print(f"[FL Paper Drawn] Generating paper drawn effect...")
|
||||
effect_frames = []
|
||||
frame_time = 1.0 / fps
|
||||
|
||||
pbar = ProgressBar(max_frames * 2) # Double progress for two passes
|
||||
|
||||
for frame_idx in range(max_frames):
|
||||
frame = image[frame_idx]
|
||||
frame_pil = self.t2p(frame)
|
||||
|
||||
# Process with shader using static parameters
|
||||
effect_img = self.process_image(
|
||||
frame_pil,
|
||||
angle_num,
|
||||
samp_num,
|
||||
line_width,
|
||||
vignette,
|
||||
(frame_idx + 1) * frame_time
|
||||
)
|
||||
|
||||
# Apply mask to this frame's effect if provided
|
||||
if mask_images is not None:
|
||||
mask_img = self.process_mask(mask_images[frame_idx], effect_img.size)
|
||||
# Blend original and effect based on mask
|
||||
effect_array = np.array(effect_img).astype(np.float32)
|
||||
original_array = np.array(frame_pil).astype(np.float32)
|
||||
mask_array = np.array(mask_img).astype(np.float32) / 255.0
|
||||
|
||||
# Expand mask to 3 channels
|
||||
if len(mask_array.shape) == 2:
|
||||
mask_array = np.stack([mask_array] * 3, axis=-1)
|
||||
|
||||
# Blend: mask=1.0 shows effect, mask=0.0 shows original
|
||||
blended = effect_array * mask_array + original_array * (1.0 - mask_array)
|
||||
effect_img = Image.fromarray(blended.astype(np.uint8))
|
||||
|
||||
effect_tensor = self.p2t(effect_img)
|
||||
effect_frames.append(effect_tensor)
|
||||
|
||||
if frame_idx % 50 == 0:
|
||||
print(f"[FL Paper Drawn] Effect generation: {frame_idx}/{max_frames}")
|
||||
|
||||
pbar.update_absolute(frame_idx + 1)
|
||||
|
||||
# Stack effect frames
|
||||
effect_batch = torch.cat(effect_frames, dim=0)
|
||||
|
||||
# Second pass: Blend based on envelope
|
||||
print(f"[FL Paper Drawn] Blending with envelope...")
|
||||
result = []
|
||||
|
||||
for frame_idx in range(max_frames):
|
||||
# Get envelope value for this frame
|
||||
envelope_value = envelope_values[frame_idx]
|
||||
|
||||
# Invert if requested
|
||||
if invert:
|
||||
blend_amount = (1.0 - envelope_value) * blend_intensity
|
||||
else:
|
||||
blend_amount = envelope_value * blend_intensity
|
||||
|
||||
# Clamp blend amount to 0-1 range
|
||||
blend_amount = max(0.0, min(1.0, blend_amount))
|
||||
|
||||
# Get original and effect frames
|
||||
original_frame = image[frame_idx]
|
||||
effect_frame = effect_batch[frame_idx]
|
||||
|
||||
# Blend: (1 - blend) * original + blend * effect
|
||||
blended_frame = (1.0 - blend_amount) * original_frame + blend_amount * effect_frame
|
||||
|
||||
result.append(blended_frame)
|
||||
|
||||
if frame_idx % 100 == 0 or frame_idx < 5:
|
||||
print(f"[FL Paper Drawn] Frame {frame_idx}: envelope={envelope_value:.3f}, blend={blend_amount:.3f}")
|
||||
|
||||
pbar.update_absolute(max_frames + frame_idx + 1)
|
||||
|
||||
# Stack all frames
|
||||
output_tensor = torch.stack(result, dim=0)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[FL Paper Drawn] Audio-reactive processing complete!")
|
||||
print(f"[FL Paper Drawn] Output frames: {output_tensor.shape[0]}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
return (output_tensor,)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[FL Paper Drawn] ERROR in audio-reactive mode: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
print(f"[FL Paper Drawn] Falling back to static mode...")
|
||||
print(f"{'='*60}\n")
|
||||
# Fallback to static mode on error
|
||||
return self._apply_static(image, angle_num, samp_num, line_width, vignette, fps, mask)
|
||||
|
||||
def process_image(self, image, angle_num, samp_num, line_width, vignette, time):
|
||||
# Convert the PIL image to a numpy array
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
|
||||
# Create a white image for iChannel1
|
||||
white_image = np.ones((image.height, image.width, 3), dtype=np.float32)
|
||||
|
||||
# Create a PyOpenGL context
|
||||
if not glfw.init():
|
||||
raise RuntimeError("Failed to initialize GLFW")
|
||||
|
||||
glfw.window_hint(glfw.VISIBLE, glfw.FALSE)
|
||||
window = glfw.create_window(image.width, image.height, "Hidden Window", None, None)
|
||||
if not window:
|
||||
glfw.terminate()
|
||||
raise RuntimeError("Failed to create GLFW window")
|
||||
|
||||
glfw.make_context_current(window)
|
||||
|
||||
# Compile the shader program
|
||||
vertex_shader = gl.glCreateShader(gl.GL_VERTEX_SHADER)
|
||||
gl.glShaderSource(vertex_shader, VERTEX_SHADER)
|
||||
gl.glCompileShader(vertex_shader)
|
||||
|
||||
fragment_shader = gl.glCreateShader(gl.GL_FRAGMENT_SHADER)
|
||||
gl.glShaderSource(fragment_shader, FRAGMENT_SHADER)
|
||||
gl.glCompileShader(fragment_shader)
|
||||
|
||||
shader_program = gl.glCreateProgram()
|
||||
gl.glAttachShader(shader_program, vertex_shader)
|
||||
gl.glAttachShader(shader_program, fragment_shader)
|
||||
gl.glLinkProgram(shader_program)
|
||||
|
||||
gl.glUseProgram(shader_program)
|
||||
|
||||
# Set up vertex buffer object (VBO) and vertex array object (VAO)
|
||||
vertices = np.array([
|
||||
-1.0, -1.0, 0.0, 0.0, 0.0,
|
||||
1.0, -1.0, 0.0, 1.0, 0.0,
|
||||
-1.0, 1.0, 0.0, 0.0, 1.0,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0
|
||||
], dtype=np.float32)
|
||||
|
||||
vao = gl.glGenVertexArrays(1)
|
||||
gl.glBindVertexArray(vao)
|
||||
|
||||
vbo = gl.glGenBuffers(1)
|
||||
gl.glBindBuffer(gl.GL_ARRAY_BUFFER, vbo)
|
||||
gl.glBufferData(gl.GL_ARRAY_BUFFER, vertices.nbytes, vertices, gl.GL_STATIC_DRAW)
|
||||
|
||||
gl.glVertexAttribPointer(0, 3, gl.GL_FLOAT, gl.GL_FALSE, 5 * vertices.itemsize, None)
|
||||
gl.glEnableVertexAttribArray(0)
|
||||
|
||||
gl.glVertexAttribPointer(1, 2, gl.GL_FLOAT, gl.GL_FALSE, 5 * vertices.itemsize, ctypes.c_void_p(3 * vertices.itemsize))
|
||||
gl.glEnableVertexAttribArray(1)
|
||||
|
||||
# Set up textures
|
||||
texture0 = gl.glGenTextures(1)
|
||||
gl.glActiveTexture(gl.GL_TEXTURE0)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture0)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.width, image.height, 0, gl.GL_RGB, gl.GL_FLOAT, img_array)
|
||||
|
||||
texture1 = gl.glGenTextures(1)
|
||||
gl.glActiveTexture(gl.GL_TEXTURE1)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture1)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.width, image.height, 0, gl.GL_RGB, gl.GL_FLOAT, white_image)
|
||||
|
||||
# Set shader uniforms
|
||||
gl.glUniform1i(gl.glGetUniformLocation(shader_program, "iChannel0"), 0)
|
||||
gl.glUniform1i(gl.glGetUniformLocation(shader_program, "iChannel1"), 1)
|
||||
gl.glUniform3f(gl.glGetUniformLocation(shader_program, "iResolution"), image.width, image.height, 0.0)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iTime"), time)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iAngleNum"), angle_num)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iSampNum"), samp_num)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iLineWidth"), line_width)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iVignette"), vignette)
|
||||
|
||||
# Render the shader
|
||||
gl.glClear(gl.GL_COLOR_BUFFER_BIT)
|
||||
gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4)
|
||||
|
||||
# Read the rendered image from the framebuffer
|
||||
img_data = gl.glReadPixels(0, 0, image.width, image.height, gl.GL_RGB, gl.GL_FLOAT)
|
||||
img_array = np.frombuffer(img_data, dtype=np.float32).reshape((image.height, image.width, 3))
|
||||
|
||||
# Clean up OpenGL resources
|
||||
gl.glDeleteTextures(2, [texture0, texture1])
|
||||
gl.glDeleteBuffers(1, [vbo])
|
||||
gl.glDeleteVertexArrays(1, [vao])
|
||||
gl.glDeleteProgram(shader_program)
|
||||
gl.glDeleteShader(vertex_shader)
|
||||
gl.glDeleteShader(fragment_shader)
|
||||
|
||||
glfw.destroy_window(window)
|
||||
glfw.terminate()
|
||||
|
||||
# Convert the processed image back to a PIL image
|
||||
processed_image = Image.fromarray((img_array * 255).astype(np.uint8))
|
||||
|
||||
return processed_image
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import sys
|
||||
import OpenGL.GL as gl
|
||||
import glfw
|
||||
import ctypes
|
||||
|
||||
|
||||
VERTEX_SHADER = """
|
||||
#version 330 core
|
||||
|
||||
layout (location = 0) in vec3 aPos;
|
||||
layout (location = 1) in vec2 aTexCoord;
|
||||
|
||||
out vec2 TexCoord;
|
||||
|
||||
void main()
|
||||
{
|
||||
gl_Position = vec4(aPos, 1.0);
|
||||
TexCoord = aTexCoord;
|
||||
}
|
||||
"""
|
||||
|
||||
FRAGMENT_SHADER = """
|
||||
#version 330 core
|
||||
|
||||
out vec4 FragColor;
|
||||
|
||||
in vec2 TexCoord;
|
||||
|
||||
uniform sampler2D iChannel0;
|
||||
uniform sampler2D iChannel1;
|
||||
uniform vec3 iResolution;
|
||||
uniform float iTime;
|
||||
uniform float iAngleNum;
|
||||
uniform float iSampNum;
|
||||
uniform float iLineWidth;
|
||||
uniform float iVignette;
|
||||
|
||||
#define Res0 textureSize(iChannel0, 0)
|
||||
#define Res1 textureSize(iChannel1, 0)
|
||||
#define Res iResolution.xy
|
||||
|
||||
#define randSamp iChannel1
|
||||
#define colorSamp iChannel0
|
||||
|
||||
vec4 getRand(vec2 pos)
|
||||
{
|
||||
return textureLod(iChannel1, pos / Res1 / iResolution.y * 1080., 0.0);
|
||||
}
|
||||
|
||||
vec4 getCol(vec2 pos)
|
||||
{
|
||||
vec2 uv = ((pos - Res.xy * .5) / Res.y * Res0.y) / Res0.xy + .5;
|
||||
vec4 c1 = texture(iChannel0, uv);
|
||||
vec4 e = smoothstep(vec4(-0.05), vec4(-0.0), vec4(uv, vec2(1) - uv));
|
||||
c1 = mix(vec4(1, 1, 1, 0), c1, e.x * e.y * e.z * e.w);
|
||||
float d = clamp(dot(c1.xyz, vec3(-.5, 1., -.5)), 0.0, 1.0);
|
||||
vec4 c2 = vec4(.7);
|
||||
return min(mix(c1, c2, 1.8 * d), .7);
|
||||
}
|
||||
|
||||
vec4 getColHT(vec2 pos)
|
||||
{
|
||||
return smoothstep(.95, 1.05, getCol(pos) * .8 + .2 + getRand(pos * .7));
|
||||
}
|
||||
|
||||
float getVal(vec2 pos)
|
||||
{
|
||||
vec4 c = getCol(pos);
|
||||
return pow(dot(c.xyz, vec3(.333)), 1.) * 1.;
|
||||
}
|
||||
|
||||
vec2 getGrad(vec2 pos, float eps)
|
||||
{
|
||||
vec2 d = vec2(eps, 0);
|
||||
return vec2(
|
||||
getVal(pos + d.xy) - getVal(pos - d.xy),
|
||||
getVal(pos + d.yx) - getVal(pos - d.yx)
|
||||
) / eps / 2.;
|
||||
}
|
||||
|
||||
#define PI2 6.28318530717959
|
||||
|
||||
void main()
|
||||
{
|
||||
vec2 pos = TexCoord * iResolution.xy + 4.0 * sin(iTime * 1. * vec2(1, 1.7)) * iResolution.y / 400.;
|
||||
vec3 col = vec3(0);
|
||||
vec3 col2 = vec3(0);
|
||||
float sum = 0.;
|
||||
for (int i = 0; i < int(iAngleNum); i++)
|
||||
{
|
||||
float ang = PI2 / iAngleNum * (float(i) + .8);
|
||||
vec2 v = vec2(cos(ang), sin(ang));
|
||||
for (int j = 0; j < int(iSampNum); j++)
|
||||
{
|
||||
vec2 dpos = v.yx * vec2(1, -1) * float(j) * iLineWidth * iResolution.y / 400.;
|
||||
vec2 dpos2 = v.xy * float(j * j) / iSampNum * .5 * iLineWidth * iResolution.y / 400.;
|
||||
vec2 g;
|
||||
float fact;
|
||||
float fact2;
|
||||
|
||||
for (float s = -1.; s <= 1.; s += 2.)
|
||||
{
|
||||
vec2 pos2 = pos + s * dpos + dpos2;
|
||||
vec2 pos3 = pos + (s * dpos + dpos2).yx * vec2(1, -1) * 2.;
|
||||
g = getGrad(pos2, .4);
|
||||
fact = dot(g, v) - .5 * abs(dot(g, v.yx * vec2(1, -1)));
|
||||
fact2 = dot(normalize(g + vec2(.0001)), v.yx * vec2(1, -1));
|
||||
|
||||
fact = clamp(fact, 0., .05);
|
||||
fact2 = abs(fact2);
|
||||
|
||||
fact *= 1. - float(j) / iSampNum;
|
||||
col += fact;
|
||||
col2 += fact2 * getColHT(pos3).xyz;
|
||||
sum += fact2;
|
||||
}
|
||||
}
|
||||
}
|
||||
col /= iSampNum * iAngleNum * .75 / sqrt(iResolution.y);
|
||||
col2 /= sum;
|
||||
col.x *= (.6 + .8 * getRand(pos * .7).x);
|
||||
col.x = 1. - col.x;
|
||||
col.x *= col.x * col.x;
|
||||
|
||||
vec2 s = sin(pos.xy * .1 / sqrt(iResolution.y / 400.));
|
||||
vec3 karo = vec3(1);
|
||||
karo -= .5 * vec3(.25, .1, .1) * dot(exp(-s * s * 80.), vec2(1));
|
||||
float r = length(pos - iResolution.xy * .5) / iResolution.x;
|
||||
float vign = 1. - r * r * r * iVignette;
|
||||
FragColor = vec4(vec3(col.x * col2 * karo * vign), 1);
|
||||
}
|
||||
"""
|
||||
|
||||
class FL_PaperDrawn:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"angle_num": ("FLOAT", {"default": 3.0, "min": 1.0, "max": 10.0, "step": 1.0}),
|
||||
"samp_num": ("FLOAT", {"default": 2.2, "min": 1.0, "max": 10.0, "step": 0.1}),
|
||||
"line_width": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}),
|
||||
"vignette": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.1}),
|
||||
"fps": ("INT", {"default": 30, "min": 1, "max": 120, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "apply_shader"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def apply_shader(self, image, angle_num, samp_num, line_width, vignette, fps):
|
||||
result = []
|
||||
total_images = len(image)
|
||||
frame_time = 1.0 / fps
|
||||
|
||||
for i, img in enumerate(image, start=1):
|
||||
img = self.t2p(img)
|
||||
result_img = self.process_image(img, angle_num, samp_num, line_width, vignette, i * frame_time)
|
||||
result_img = self.p2t(result_img)
|
||||
result.append(result_img)
|
||||
|
||||
# Update the print log
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress log
|
||||
print()
|
||||
|
||||
return (torch.cat(result, dim=0),)
|
||||
|
||||
def process_image(self, image, angle_num, samp_num, line_width, vignette, time):
|
||||
# Convert the PIL image to a numpy array
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
|
||||
# Create a white image for iChannel1
|
||||
white_image = np.ones((image.height, image.width, 3), dtype=np.float32)
|
||||
|
||||
# Create a PyOpenGL context
|
||||
if not glfw.init():
|
||||
raise RuntimeError("Failed to initialize GLFW")
|
||||
|
||||
glfw.window_hint(glfw.VISIBLE, glfw.FALSE)
|
||||
window = glfw.create_window(image.width, image.height, "Hidden Window", None, None)
|
||||
if not window:
|
||||
glfw.terminate()
|
||||
raise RuntimeError("Failed to create GLFW window")
|
||||
|
||||
glfw.make_context_current(window)
|
||||
|
||||
# Compile the shader program
|
||||
vertex_shader = gl.glCreateShader(gl.GL_VERTEX_SHADER)
|
||||
gl.glShaderSource(vertex_shader, VERTEX_SHADER)
|
||||
gl.glCompileShader(vertex_shader)
|
||||
|
||||
fragment_shader = gl.glCreateShader(gl.GL_FRAGMENT_SHADER)
|
||||
gl.glShaderSource(fragment_shader, FRAGMENT_SHADER)
|
||||
gl.glCompileShader(fragment_shader)
|
||||
|
||||
shader_program = gl.glCreateProgram()
|
||||
gl.glAttachShader(shader_program, vertex_shader)
|
||||
gl.glAttachShader(shader_program, fragment_shader)
|
||||
gl.glLinkProgram(shader_program)
|
||||
|
||||
gl.glUseProgram(shader_program)
|
||||
|
||||
# Set up vertex buffer object (VBO) and vertex array object (VAO)
|
||||
vertices = np.array([
|
||||
-1.0, -1.0, 0.0, 0.0, 0.0,
|
||||
1.0, -1.0, 0.0, 1.0, 0.0,
|
||||
-1.0, 1.0, 0.0, 0.0, 1.0,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0
|
||||
], dtype=np.float32)
|
||||
|
||||
vao = gl.glGenVertexArrays(1)
|
||||
gl.glBindVertexArray(vao)
|
||||
|
||||
vbo = gl.glGenBuffers(1)
|
||||
gl.glBindBuffer(gl.GL_ARRAY_BUFFER, vbo)
|
||||
gl.glBufferData(gl.GL_ARRAY_BUFFER, vertices.nbytes, vertices, gl.GL_STATIC_DRAW)
|
||||
|
||||
gl.glVertexAttribPointer(0, 3, gl.GL_FLOAT, gl.GL_FALSE, 5 * vertices.itemsize, None)
|
||||
gl.glEnableVertexAttribArray(0)
|
||||
|
||||
gl.glVertexAttribPointer(1, 2, gl.GL_FLOAT, gl.GL_FALSE, 5 * vertices.itemsize, ctypes.c_void_p(3 * vertices.itemsize))
|
||||
gl.glEnableVertexAttribArray(1)
|
||||
|
||||
# Set up textures
|
||||
texture0 = gl.glGenTextures(1)
|
||||
gl.glActiveTexture(gl.GL_TEXTURE0)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture0)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.width, image.height, 0, gl.GL_RGB, gl.GL_FLOAT, img_array)
|
||||
|
||||
texture1 = gl.glGenTextures(1)
|
||||
gl.glActiveTexture(gl.GL_TEXTURE1)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture1)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_REPEAT)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.width, image.height, 0, gl.GL_RGB, gl.GL_FLOAT, white_image)
|
||||
|
||||
# Set shader uniforms
|
||||
gl.glUniform1i(gl.glGetUniformLocation(shader_program, "iChannel0"), 0)
|
||||
gl.glUniform1i(gl.glGetUniformLocation(shader_program, "iChannel1"), 1)
|
||||
gl.glUniform3f(gl.glGetUniformLocation(shader_program, "iResolution"), image.width, image.height, 0.0)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iTime"), time)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iAngleNum"), angle_num)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iSampNum"), samp_num)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iLineWidth"), line_width)
|
||||
gl.glUniform1f(gl.glGetUniformLocation(shader_program, "iVignette"), vignette)
|
||||
|
||||
# Render the shader
|
||||
gl.glClear(gl.GL_COLOR_BUFFER_BIT)
|
||||
gl.glDrawArrays(gl.GL_TRIANGLE_STRIP, 0, 4)
|
||||
|
||||
# Read the rendered image from the framebuffer
|
||||
img_data = gl.glReadPixels(0, 0, image.width, image.height, gl.GL_RGB, gl.GL_FLOAT)
|
||||
img_array = np.frombuffer(img_data, dtype=np.float32).reshape((image.height, image.width, 3))
|
||||
|
||||
# Clean up OpenGL resources
|
||||
gl.glDeleteTextures(2, [texture0, texture1])
|
||||
gl.glDeleteBuffers(1, [vbo])
|
||||
gl.glDeleteVertexArrays(1, [vao])
|
||||
gl.glDeleteProgram(shader_program)
|
||||
gl.glDeleteShader(vertex_shader)
|
||||
gl.glDeleteShader(fragment_shader)
|
||||
|
||||
glfw.destroy_window(window)
|
||||
glfw.terminate()
|
||||
|
||||
# Convert the processed image back to a PIL image
|
||||
processed_image = Image.fromarray((img_array * 255).astype(np.uint8))
|
||||
|
||||
return processed_image
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def p2t(self, p):
|
||||
if p is not None:
|
||||
i = np.array(p).astype(np.float32) / 255.0
|
||||
t = torch.from_numpy(i).unsqueeze(0)
|
||||
return t
|
||||
@@ -0,0 +1,100 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import sys
|
||||
|
||||
class FL_PixelArtShader:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"pixel_size": ("FLOAT", {"default": 100.0, "min": 1.0, "max": 1000.0, "step": 1.0}),
|
||||
"color_depth": ("FLOAT", {"default": 50.0, "min": 1.0, "max": 255.0, "step": 1.0}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "apply_pixel_art_shader"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def apply_pixel_art_shader(self, images, pixel_size, color_depth):
|
||||
result = []
|
||||
total_images = len(images)
|
||||
|
||||
for i, image in enumerate(images, start=1):
|
||||
img = self.t2p(image)
|
||||
result_img = pixel_art_effect(img, pixel_size, color_depth)
|
||||
result_img = self.p2t(result_img)
|
||||
result.append(result_img)
|
||||
|
||||
# Update the print log
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress log
|
||||
print()
|
||||
|
||||
return (torch.cat(result, dim=0),)
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def p2t(self, p):
|
||||
if p is not None:
|
||||
i = np.array(p).astype(np.float32) / 255.0
|
||||
t = torch.from_numpy(i).unsqueeze(0)
|
||||
return t
|
||||
|
||||
def pixel_art_effect(image, pixel_size, color_depth):
|
||||
# Move the input image to the GPU
|
||||
image = torch.tensor(np.array(image)).float().to("cuda") / 255.0
|
||||
|
||||
# Get the input image dimensions
|
||||
height, width = image.shape[0], image.shape[1]
|
||||
|
||||
# Create a grid of UV coordinates
|
||||
uv_x = torch.linspace(0, 1, width, device="cuda")
|
||||
uv_y = torch.linspace(0, 1, height, device="cuda")
|
||||
uv_grid = torch.stack(torch.meshgrid(uv_y, uv_x), dim=-1) # Swap uv_y and uv_x
|
||||
|
||||
# Evaluate the shader code for each pixel
|
||||
output_tensor = evaluate_shader(image, uv_grid, pixel_size, color_depth)
|
||||
|
||||
# Convert the output tensor to a PIL image
|
||||
output_image = Image.fromarray((output_tensor.cpu().numpy() * 255).astype(np.uint8))
|
||||
|
||||
return output_image
|
||||
|
||||
def evaluate_shader(image, uv_grid, pixel_size, color_depth):
|
||||
# Sample the input image at the pixelated UV coordinates
|
||||
pixelUV = torch.floor(uv_grid * pixel_size) / pixel_size
|
||||
color = texture_lookup(image, pixelUV)
|
||||
|
||||
# Apply color adjustments
|
||||
color = adjust_color(color, color_depth)
|
||||
|
||||
return color
|
||||
|
||||
def adjust_color(color, color_depth):
|
||||
# Apply color depth reduction
|
||||
color = torch.floor(color * color_depth) / color_depth
|
||||
|
||||
return color
|
||||
|
||||
def texture_lookup(image, uv):
|
||||
# Clamp the UV coordinates to [0, 1]
|
||||
uv = torch.clamp(uv, 0.0, 1.0)
|
||||
|
||||
# Calculate the pixel coordinates
|
||||
y = (uv[..., 0] * (image.shape[0] - 1)).long() # Use uv[..., 0] for y
|
||||
x = (uv[..., 1] * (image.shape[1] - 1)).long() # Use uv[..., 1] for x
|
||||
|
||||
# Perform texture lookup using the pixel coordinates
|
||||
return image[y, x]
|
||||
@@ -1,75 +1,79 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from colorsys import rgb_to_hsv
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
class FL_PixelSort:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"direction": (["Horizontal", "Vertical"],),
|
||||
"threshold": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"smoothing": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"rotation": ("INT", {"default": 0, "min": 0, "max": 3, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "pixel_sort_saturation"
|
||||
CATEGORY = "🏵️Fill Nodes/VFX"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def saturation(self, pixel):
|
||||
r, g, b = pixel
|
||||
_, s, _ = rgb_to_hsv(r / 255.0, g / 255.0, b / 255.0)
|
||||
return s
|
||||
|
||||
def pixel_sort_saturation(self, images, direction="Horizontal", threshold=0.5, smoothing=0.1, rotation=0):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
pbar = ProgressBar(total_images)
|
||||
for i, img in enumerate(images, start=1):
|
||||
p = self.t2p(img)
|
||||
sorted_image = self.sort_pixels(p, self.saturation, threshold, smoothing, rotation)
|
||||
o = np.array(sorted_image.convert("RGB")).astype(np.float32) / 255.0
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
pbar.update_absolute(i)
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
|
||||
def sort_pixels(self, image, value, threshold, smoothing, rotation=0):
|
||||
pixels = np.rot90(np.array(image), rotation)
|
||||
values = np.apply_along_axis(value, 2, pixels)
|
||||
edges = np.apply_along_axis(lambda row: np.convolve(row, [-1, 1], 'same'), 0, values > threshold)
|
||||
edges = np.maximum(edges, 0)
|
||||
edges = np.minimum(edges, 1)
|
||||
edges = np.convolve(edges.flatten(), np.ones(int(smoothing * pixels.shape[1])), 'same').reshape(edges.shape)
|
||||
|
||||
intervals = [np.flatnonzero(row) for row in edges]
|
||||
|
||||
pbar = ProgressBar(len(values))
|
||||
for row, key in enumerate(values):
|
||||
order = np.split(key, intervals[row])
|
||||
for index, interval in enumerate(order[1:]):
|
||||
order[index + 1] = np.argsort(interval) + intervals[row][index]
|
||||
order[0] = range(order[0].size)
|
||||
order = np.concatenate(order)
|
||||
|
||||
for channel in range(3):
|
||||
pixels[row, :, channel] = pixels[row, order.astype('uint32'), channel]
|
||||
|
||||
pbar.update_absolute(row)
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from colorsys import rgb_to_hsv
|
||||
import sys
|
||||
|
||||
|
||||
class FL_PixelSort:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"direction": (["Horizontal", "Vertical"],),
|
||||
"threshold": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"smoothing": ("FLOAT", {"default": 0.1, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"rotation": ("INT", {"default": 0, "min": 0, "max": 3, "step": 1}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "pixel_sort_saturation"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def saturation(self, pixel):
|
||||
r, g, b = pixel
|
||||
_, s, _ = rgb_to_hsv(r / 255.0, g / 255.0, b / 255.0)
|
||||
return s
|
||||
|
||||
def pixel_sort_saturation(self, images, direction="Horizontal", threshold=0.5, smoothing=0.1, rotation=0):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
for i, img in enumerate(images, start=1):
|
||||
p = self.t2p(img)
|
||||
sorted_image = self.sort_pixels(p, self.saturation, threshold, smoothing, rotation)
|
||||
o = np.array(sorted_image.convert("RGB")).astype(np.float32) / 255.0
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
# Print progress update
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
|
||||
def sort_pixels(self, image, value, threshold, smoothing, rotation=0):
|
||||
pixels = np.rot90(np.array(image), rotation)
|
||||
values = np.apply_along_axis(value, 2, pixels)
|
||||
edges = np.apply_along_axis(lambda row: np.convolve(row, [-1, 1], 'same'), 0, values > threshold)
|
||||
edges = np.maximum(edges, 0)
|
||||
edges = np.minimum(edges, 1)
|
||||
edges = np.convolve(edges.flatten(), np.ones(int(smoothing * pixels.shape[1])), 'same').reshape(edges.shape)
|
||||
|
||||
intervals = [np.flatnonzero(row) for row in edges]
|
||||
|
||||
for row, key in enumerate(values):
|
||||
order = np.split(key, intervals[row])
|
||||
for index, interval in enumerate(order[1:]):
|
||||
order[index + 1] = np.argsort(interval) + intervals[row][index]
|
||||
order[0] = range(order[0].size)
|
||||
order = np.concatenate(order)
|
||||
|
||||
for channel in range(3):
|
||||
pixels[row, :, channel] = pixels[row, order.astype('uint32'), channel]
|
||||
|
||||
return Image.fromarray(np.rot90(pixels, -rotation))
|
||||
@@ -1,36 +1,36 @@
|
||||
class FL_PromptSelector:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prepend_text": ("STRING", {"multiline": True, "default": ""}),
|
||||
"prompts": ("STRING", {"multiline": True}),
|
||||
"append_text": ("STRING", {"multiline": True, "default": ""}),
|
||||
"index": ("INT", {"default": 0, "min": 0, "max": 6969, "step": 1}),
|
||||
},
|
||||
"optional": {},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "select_prompt"
|
||||
CATEGORY = "🏵️Fill Nodes/Prompting"
|
||||
|
||||
def select_prompt(self, prepend_text, prompts, append_text, index):
|
||||
prepend_text = prepend_text.strip()
|
||||
prompt_lines = prompts.split("\n")
|
||||
append_text = append_text.strip()
|
||||
|
||||
num_prompts = len(prompt_lines)
|
||||
|
||||
# Loop through prompts using modulo
|
||||
index = index % num_prompts
|
||||
|
||||
selected_prompt = prompt_lines[index].strip()
|
||||
|
||||
if prepend_text:
|
||||
selected_prompt = prepend_text + " " + selected_prompt
|
||||
|
||||
if append_text:
|
||||
selected_prompt = selected_prompt + " " + append_text
|
||||
|
||||
class FL_PromptSelector:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prepend_text": ("STRING", {"multiline": True, "default": ""}),
|
||||
"prompts": ("STRING", {"multiline": True}),
|
||||
"append_text": ("STRING", {"multiline": True, "default": ""}),
|
||||
"index": ("INT", {"default": 0, "min": 0, "max": 100, "step": 1}),
|
||||
},
|
||||
"optional": {},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "select_prompt"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def select_prompt(self, prepend_text, prompts, append_text, index):
|
||||
prepend_text = prepend_text.strip()
|
||||
prompt_lines = prompts.split("\n")
|
||||
append_text = append_text.strip()
|
||||
|
||||
num_prompts = len(prompt_lines)
|
||||
|
||||
if index < 0 or index >= num_prompts:
|
||||
raise ValueError(f"Index {index} is out of range. Please provide an index between 0 and {num_prompts - 1}.")
|
||||
|
||||
selected_prompt = prompt_lines[index].strip()
|
||||
|
||||
if prepend_text:
|
||||
selected_prompt = prepend_text + " " + selected_prompt
|
||||
|
||||
if append_text:
|
||||
selected_prompt = selected_prompt + " " + append_text
|
||||
|
||||
return (selected_prompt,)
|
||||
@@ -1,38 +1,38 @@
|
||||
import random
|
||||
import torch
|
||||
|
||||
class FL_RandomNumber:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"min_value": ("FLOAT", {"default": 0.0, "min": -1000000.0, "max": 1000000.0, "step": 0.1}),
|
||||
"max_value": ("FLOAT", {"default": 1.0, "min": -1000000.0, "max": 1000000.0, "step": 0.1}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 1000000}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("INT", "FLOAT")
|
||||
FUNCTION = "generate_random_number"
|
||||
CATEGORY = "🏵️Fill Nodes/Utility"
|
||||
|
||||
def generate_random_number(self, min_value=0.0, max_value=1.0, seed=0):
|
||||
if min_value > max_value:
|
||||
raise ValueError("min_value should be less than or equal to max_value")
|
||||
|
||||
# Generate a random seed if seed is 0
|
||||
if seed == 0:
|
||||
seed = random.randint(1, 1000000)
|
||||
|
||||
# Set the random seed for reproducibility
|
||||
random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
|
||||
# Generate a random float value within the specified range
|
||||
random_float = min_value + (max_value - min_value) * random.random()
|
||||
|
||||
# Generate a random integer value within the specified range
|
||||
random_int = int(min_value + (max_value - min_value) * random.random())
|
||||
|
||||
import random
|
||||
import torch
|
||||
|
||||
class FL_RandomNumber:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"min_value": ("FLOAT", {"default": 0.0, "min": -1000000.0, "max": 1000000.0, "step": 0.1}),
|
||||
"max_value": ("FLOAT", {"default": 1.0, "min": -1000000.0, "max": 1000000.0, "step": 0.1}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 1000000}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("INT", "FLOAT")
|
||||
FUNCTION = "generate_random_number"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def generate_random_number(self, min_value=0.0, max_value=1.0, seed=0):
|
||||
if min_value > max_value:
|
||||
raise ValueError("min_value should be less than or equal to max_value")
|
||||
|
||||
# Generate a random seed if seed is 0
|
||||
if seed == 0:
|
||||
seed = random.randint(1, 1000000)
|
||||
|
||||
# Set the random seed for reproducibility
|
||||
random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
|
||||
# Generate a random float value within the specified range
|
||||
random_float = min_value + (max_value - min_value) * random.random()
|
||||
|
||||
# Generate a random integer value within the specified range
|
||||
random_int = int(min_value + (max_value - min_value) * random.random())
|
||||
|
||||
return (random_int, random_float)
|
||||
@@ -0,0 +1,81 @@
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import math
|
||||
import sys
|
||||
|
||||
class FL_Ripple:
|
||||
def __init__(self):
|
||||
self.modulation_index = 0
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"amplitude": ("FLOAT", {"default": 10.0, "min": 0.1, "max": 50.0, "step": 0.1}),
|
||||
"frequency": ("FLOAT", {"default": 20.0, "min": 1.0, "max": 100.0, "step": 0.1}),
|
||||
"phase": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 360.0, "step": 1.0}),
|
||||
"center_x": ("FLOAT", {"default": 50.0, "min": 0.0, "max": 100.0, "step": 0.1}),
|
||||
"center_y": ("FLOAT", {"default": 50.0, "min": 0.0, "max": 100.0, "step": 0.1}),
|
||||
"modulation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "ripple"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def t2p(self, t):
|
||||
if t is not None:
|
||||
i = 255.0 * t.cpu().numpy().squeeze()
|
||||
p = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
return p
|
||||
|
||||
def ripple(self, images, amplitude=10.0, frequency=20.0, phase=0.0, center_x=50.0, center_y=50.0, modulation=0.0):
|
||||
out = []
|
||||
total_images = len(images)
|
||||
for i, img in enumerate(images, start=1):
|
||||
p = self.t2p(img)
|
||||
width, height = p.size
|
||||
center_x_pixel = int(center_x / 100 * width)
|
||||
center_y_pixel = int(center_y / 100 * height)
|
||||
|
||||
x, y = np.meshgrid(np.arange(width), np.arange(height))
|
||||
dx = x - center_x_pixel
|
||||
dy = y - center_y_pixel
|
||||
distance = np.sqrt(dx ** 2 + dy ** 2)
|
||||
|
||||
# Apply modulation to amplitude and frequency
|
||||
modulation_factor = 1 + modulation * math.sin(2 * math.pi * self.modulation_index / total_images)
|
||||
modulated_amplitude = amplitude * modulation_factor
|
||||
modulated_frequency = frequency * modulation_factor
|
||||
|
||||
angle = distance / modulated_frequency * 2 * np.pi + np.radians(phase)
|
||||
offset_x = (modulated_amplitude * np.sin(angle)).astype(int)
|
||||
offset_y = (modulated_amplitude * np.cos(angle)).astype(int)
|
||||
|
||||
sample_x = np.clip(x + offset_x, 0, width - 1)
|
||||
sample_y = np.clip(y + offset_y, 0, height - 1)
|
||||
|
||||
p_array = np.array(p)
|
||||
rippled_array = p_array[sample_y, sample_x]
|
||||
|
||||
o = rippled_array.astype(np.float32) / 255.0
|
||||
o = torch.from_numpy(o).unsqueeze(0)
|
||||
out.append(o)
|
||||
|
||||
self.modulation_index += 1
|
||||
|
||||
# Print progress update
|
||||
progress = i / total_images * 100
|
||||
sys.stdout.write(f"\rProcessing images: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
out = torch.cat(out, 0)
|
||||
return (out,)
|
||||
@@ -1,221 +1,229 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
import OpenGL.GL as gl
|
||||
import glfw
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
SHADERTOY_HEADER = """
|
||||
#version 440
|
||||
|
||||
precision highp float;
|
||||
|
||||
uniform vec3 iResolution;
|
||||
uniform vec4 iMouse;
|
||||
uniform float iTime;
|
||||
uniform float iTimeDelta;
|
||||
uniform float iFrameRate;
|
||||
uniform int iFrame;
|
||||
|
||||
uniform sampler2D iChannel0;
|
||||
uniform sampler2D iChannel1;
|
||||
uniform sampler2D iChannel2;
|
||||
uniform sampler2D iChannel3;
|
||||
|
||||
#define texture2D texture
|
||||
|
||||
"""
|
||||
|
||||
SHADERTOY_FOOTER = """
|
||||
|
||||
layout(location = 0) out vec4 _fragColor;
|
||||
|
||||
void main()
|
||||
{
|
||||
mainImage(_fragColor, gl_FragCoord.xy);
|
||||
}
|
||||
"""
|
||||
|
||||
SHADERTOY_DEFAULT = """
|
||||
void mainImage( out vec4 fragColor, in vec2 fragCoord )
|
||||
{
|
||||
// Normalized pixel coordinates (from 0 to 1)
|
||||
vec2 uv = fragCoord/iResolution.xy;
|
||||
|
||||
// Time varying pixel color
|
||||
vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4));
|
||||
|
||||
// Output to screen
|
||||
fragColor = vec4(col,1.0);
|
||||
}
|
||||
"""
|
||||
|
||||
def render_surface_and_context_init(width, height):
|
||||
if not glfw.init():
|
||||
raise RuntimeError("GLFW did not init")
|
||||
|
||||
glfw.window_hint(glfw.VISIBLE, glfw.FALSE) # hidden
|
||||
window = glfw.create_window(width, height, "hidden", None, None)
|
||||
if not window:
|
||||
raise RuntimeError("GLFW did not init window")
|
||||
|
||||
glfw.make_context_current(window)
|
||||
return {}
|
||||
|
||||
def render_surface_and_context_deinit(**kwargs):
|
||||
glfw.terminate()
|
||||
|
||||
def compile_shader(source, shader_type):
|
||||
shader = gl.glCreateShader(shader_type)
|
||||
gl.glShaderSource(shader, source)
|
||||
gl.glCompileShader(shader)
|
||||
if gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS) != gl.GL_TRUE:
|
||||
raise RuntimeError(gl.glGetShaderInfoLog(shader))
|
||||
return shader
|
||||
|
||||
def compile_program(vertex_source, fragment_source):
|
||||
vertex_shader = compile_shader(vertex_source, gl.GL_VERTEX_SHADER)
|
||||
fragment_shader = compile_shader(fragment_source, gl.GL_FRAGMENT_SHADER)
|
||||
program = gl.glCreateProgram()
|
||||
gl.glAttachShader(program, vertex_shader)
|
||||
gl.glAttachShader(program, fragment_shader)
|
||||
gl.glLinkProgram(program)
|
||||
if gl.glGetProgramiv(program, gl.GL_LINK_STATUS) != gl.GL_TRUE:
|
||||
raise RuntimeError(gl.glGetProgramInfoLog(program))
|
||||
return program
|
||||
|
||||
def setup_framebuffer(width, height):
|
||||
texture = gl.glGenTextures(1)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, width, height, 0, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, None)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
|
||||
fbo = gl.glGenFramebuffers(1)
|
||||
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo)
|
||||
gl.glFramebufferTexture2D(gl.GL_FRAMEBUFFER, gl.GL_COLOR_ATTACHMENT0, gl.GL_TEXTURE_2D, texture, 0)
|
||||
if gl.glCheckFramebufferStatus(gl.GL_FRAMEBUFFER) != gl.GL_FRAMEBUFFER_COMPLETE:
|
||||
raise RuntimeError("Framebuffer is not complete")
|
||||
|
||||
return fbo, texture
|
||||
|
||||
def setup_render_resources(width, height, fragment_source: str):
|
||||
ctx = render_surface_and_context_init(width, height)
|
||||
|
||||
vertex_source = """
|
||||
#version 330 core
|
||||
void main()
|
||||
{
|
||||
vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3));
|
||||
gl_Position = vec4(verts[gl_VertexID], 0, 1);
|
||||
}
|
||||
"""
|
||||
shader = compile_program(vertex_source, fragment_source)
|
||||
|
||||
fbo, texture = setup_framebuffer(width, height)
|
||||
|
||||
textures = gl.glGenTextures(4)
|
||||
|
||||
return (ctx, fbo, shader, textures)
|
||||
|
||||
def render_resources_cleanup(ctx):
|
||||
# assume all other resources get cleaned up with the context
|
||||
render_surface_and_context_deinit(**ctx)
|
||||
|
||||
def render(width, height, fbo, shader):
|
||||
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo)
|
||||
gl.glClearColor(0.0, 0.0, 0.0, 1.0)
|
||||
gl.glClear(gl.GL_COLOR_BUFFER_BIT)
|
||||
|
||||
gl.glUseProgram(shader)
|
||||
gl.glDrawArrays(gl.GL_TRIANGLES, 0, 3)
|
||||
|
||||
data = gl.glReadPixels(0, 0, width, height, gl.GL_RGB, gl.GL_UNSIGNED_BYTE)
|
||||
image = np.frombuffer(data, dtype=np.uint8).reshape(height, width, 3)
|
||||
image = image[::-1, :, :]
|
||||
image = np.array(image).astype(np.float32) / 255.0
|
||||
|
||||
return image
|
||||
|
||||
def shadertoy_vars_update(shader, width, height, time, time_delta, frame_rate, frame):
|
||||
gl.glUseProgram(shader)
|
||||
iResolution_location = gl.glGetUniformLocation(shader, "iResolution")
|
||||
gl.glUniform3f(iResolution_location, width, height, 0)
|
||||
iMouse_location = gl.glGetUniformLocation(shader, "iMouse")
|
||||
gl.glUniform4f(iMouse_location, 0, 0, 0, 0)
|
||||
iTime_location = gl.glGetUniformLocation(shader, "iTime")
|
||||
gl.glUniform1f(iTime_location, time)
|
||||
iTimeDelta_location = gl.glGetUniformLocation(shader, "iTimeDelta")
|
||||
gl.glUniform1f(iTimeDelta_location, time_delta)
|
||||
iFrameRate_location = gl.glGetUniformLocation(shader, "iFrameRate")
|
||||
gl.glUniform1f(iFrameRate_location, frame_rate)
|
||||
iFrame_location = gl.glGetUniformLocation(shader, "iFrame")
|
||||
gl.glUniform1i(iFrame_location, frame)
|
||||
|
||||
def shadertoy_texture_update(texture, image, frame):
|
||||
if len(image.shape) == 4:
|
||||
image = image[frame]
|
||||
image = image.cpu().numpy()
|
||||
image = image[::-1, :, :]
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.shape[1], image.shape[0], 0, gl.GL_RGB, gl.GL_FLOAT, image)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE)
|
||||
|
||||
def shadertoy_texture_bind(shader, textures):
|
||||
gl.glUseProgram(shader)
|
||||
for i in range(4):
|
||||
gl.glActiveTexture(gl.GL_TEXTURE0 + i) # type: ignore
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, textures[i])
|
||||
iChannel_location = gl.glGetUniformLocation(shader, f"iChannel{i}")
|
||||
gl.glUniform1i(iChannel_location, i)
|
||||
|
||||
class FL_Shadertoy:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {"required": {"width": ("INT", {"default": 512, "min": 64, "max": 15360, "step": 8}),
|
||||
"height": ("INT", {"default": 512, "min": 64, "max": 15360, "step": 8}),
|
||||
"frame_count": ("INT", {"default": 1, "min": 1, "max": 262144}),
|
||||
"fps": ("INT", {"default": 1, "min": 1, "max": 120}),
|
||||
"source": (
|
||||
"STRING", {"default": SHADERTOY_DEFAULT, "multiline": True, "dynamicPrompts": False})},
|
||||
"optional": {"channel_0": ("IMAGE",),
|
||||
"channel_1": ("IMAGE",),
|
||||
"channel_2": ("IMAGE",),
|
||||
"channel_3": ("IMAGE",)}}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
CATEGORY = "🏵️Fill Nodes/VFX"
|
||||
FUNCTION = "render"
|
||||
|
||||
def render(self, width: int, height: int, frame_count: int, fps: int, source: str,
|
||||
channel_0: torch.Tensor | None = None, channel_1: torch.Tensor | None = None,
|
||||
channel_2: torch.Tensor | None = None, channel_3: torch.Tensor | None = None):
|
||||
fragment_source = SHADERTOY_HEADER
|
||||
fragment_source += source
|
||||
fragment_source += SHADERTOY_FOOTER
|
||||
|
||||
ctx, fbo, shader, textures = setup_render_resources(width, height, fragment_source)
|
||||
|
||||
images = []
|
||||
frame = 0
|
||||
pbar = ProgressBar(frame_count)
|
||||
for idx in range(frame_count):
|
||||
shadertoy_vars_update(shader, width, height, frame * (1.0 / fps), (1.0 / fps), fps, frame)
|
||||
if channel_0 is not None: shadertoy_texture_update(textures[0], channel_0, frame)
|
||||
if channel_1 is not None: shadertoy_texture_update(textures[1], channel_1, frame)
|
||||
if channel_2 is not None: shadertoy_texture_update(textures[2], channel_2, frame)
|
||||
if channel_3 is not None: shadertoy_texture_update(textures[3], channel_3, frame)
|
||||
shadertoy_texture_bind(shader, textures)
|
||||
|
||||
image = render(width, height, fbo, shader)
|
||||
image = torch.from_numpy(image)[None,]
|
||||
images.append(image)
|
||||
|
||||
frame += 1
|
||||
pbar.update_absolute(idx)
|
||||
|
||||
render_resources_cleanup(ctx)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import os
|
||||
import sys
|
||||
import OpenGL.GL as gl
|
||||
import glfw
|
||||
|
||||
SHADERTOY_HEADER = """
|
||||
#version 440
|
||||
|
||||
precision highp float;
|
||||
|
||||
uniform vec3 iResolution;
|
||||
uniform vec4 iMouse;
|
||||
uniform float iTime;
|
||||
uniform float iTimeDelta;
|
||||
uniform float iFrameRate;
|
||||
uniform int iFrame;
|
||||
|
||||
uniform sampler2D iChannel0;
|
||||
uniform sampler2D iChannel1;
|
||||
uniform sampler2D iChannel2;
|
||||
uniform sampler2D iChannel3;
|
||||
|
||||
#define texture2D texture
|
||||
|
||||
"""
|
||||
|
||||
SHADERTOY_FOOTER = """
|
||||
|
||||
layout(location = 0) out vec4 _fragColor;
|
||||
|
||||
void main()
|
||||
{
|
||||
mainImage(_fragColor, gl_FragCoord.xy);
|
||||
}
|
||||
"""
|
||||
|
||||
SHADERTOY_DEFAULT = """
|
||||
void mainImage( out vec4 fragColor, in vec2 fragCoord )
|
||||
{
|
||||
// Normalized pixel coordinates (from 0 to 1)
|
||||
vec2 uv = fragCoord/iResolution.xy;
|
||||
|
||||
// Time varying pixel color
|
||||
vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4));
|
||||
|
||||
// Output to screen
|
||||
fragColor = vec4(col,1.0);
|
||||
}
|
||||
"""
|
||||
|
||||
def render_surface_and_context_init(width, height):
|
||||
if not glfw.init():
|
||||
raise RuntimeError("GLFW did not init")
|
||||
|
||||
glfw.window_hint(glfw.VISIBLE, glfw.FALSE) # hidden
|
||||
window = glfw.create_window(width, height, "hidden", None, None)
|
||||
if not window:
|
||||
raise RuntimeError("GLFW did not init window")
|
||||
|
||||
glfw.make_context_current(window)
|
||||
return {}
|
||||
|
||||
def render_surface_and_context_deinit(**kwargs):
|
||||
glfw.terminate()
|
||||
|
||||
def compile_shader(source, shader_type):
|
||||
shader = gl.glCreateShader(shader_type)
|
||||
gl.glShaderSource(shader, source)
|
||||
gl.glCompileShader(shader)
|
||||
if gl.glGetShaderiv(shader, gl.GL_COMPILE_STATUS) != gl.GL_TRUE:
|
||||
raise RuntimeError(gl.glGetShaderInfoLog(shader))
|
||||
return shader
|
||||
|
||||
def compile_program(vertex_source, fragment_source):
|
||||
vertex_shader = compile_shader(vertex_source, gl.GL_VERTEX_SHADER)
|
||||
fragment_shader = compile_shader(fragment_source, gl.GL_FRAGMENT_SHADER)
|
||||
program = gl.glCreateProgram()
|
||||
gl.glAttachShader(program, vertex_shader)
|
||||
gl.glAttachShader(program, fragment_shader)
|
||||
gl.glLinkProgram(program)
|
||||
if gl.glGetProgramiv(program, gl.GL_LINK_STATUS) != gl.GL_TRUE:
|
||||
raise RuntimeError(gl.glGetProgramInfoLog(program))
|
||||
return program
|
||||
|
||||
def setup_framebuffer(width, height):
|
||||
texture = gl.glGenTextures(1)
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, width, height, 0, gl.GL_RGB, gl.GL_UNSIGNED_BYTE, None)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
|
||||
fbo = gl.glGenFramebuffers(1)
|
||||
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo)
|
||||
gl.glFramebufferTexture2D(gl.GL_FRAMEBUFFER, gl.GL_COLOR_ATTACHMENT0, gl.GL_TEXTURE_2D, texture, 0)
|
||||
if gl.glCheckFramebufferStatus(gl.GL_FRAMEBUFFER) != gl.GL_FRAMEBUFFER_COMPLETE:
|
||||
raise RuntimeError("Framebuffer is not complete")
|
||||
|
||||
return fbo, texture
|
||||
|
||||
def setup_render_resources(width, height, fragment_source: str):
|
||||
ctx = render_surface_and_context_init(width, height)
|
||||
|
||||
vertex_source = """
|
||||
#version 330 core
|
||||
void main()
|
||||
{
|
||||
vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3));
|
||||
gl_Position = vec4(verts[gl_VertexID], 0, 1);
|
||||
}
|
||||
"""
|
||||
shader = compile_program(vertex_source, fragment_source)
|
||||
|
||||
fbo, texture = setup_framebuffer(width, height)
|
||||
|
||||
textures = gl.glGenTextures(4)
|
||||
|
||||
return (ctx, fbo, shader, textures)
|
||||
|
||||
def render_resources_cleanup(ctx):
|
||||
# assume all other resources get cleaned up with the context
|
||||
render_surface_and_context_deinit(**ctx)
|
||||
|
||||
def render(width, height, fbo, shader):
|
||||
gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, fbo)
|
||||
gl.glClearColor(0.0, 0.0, 0.0, 1.0)
|
||||
gl.glClear(gl.GL_COLOR_BUFFER_BIT)
|
||||
|
||||
gl.glUseProgram(shader)
|
||||
gl.glDrawArrays(gl.GL_TRIANGLES, 0, 3)
|
||||
|
||||
data = gl.glReadPixels(0, 0, width, height, gl.GL_RGB, gl.GL_UNSIGNED_BYTE)
|
||||
image = np.frombuffer(data, dtype=np.uint8).reshape(height, width, 3)
|
||||
image = image[::-1, :, :]
|
||||
image = np.array(image).astype(np.float32) / 255.0
|
||||
|
||||
return image
|
||||
|
||||
def shadertoy_vars_update(shader, width, height, time, time_delta, frame_rate, frame):
|
||||
gl.glUseProgram(shader)
|
||||
iResolution_location = gl.glGetUniformLocation(shader, "iResolution")
|
||||
gl.glUniform3f(iResolution_location, width, height, 0)
|
||||
iMouse_location = gl.glGetUniformLocation(shader, "iMouse")
|
||||
gl.glUniform4f(iMouse_location, 0, 0, 0, 0)
|
||||
iTime_location = gl.glGetUniformLocation(shader, "iTime")
|
||||
gl.glUniform1f(iTime_location, time)
|
||||
iTimeDelta_location = gl.glGetUniformLocation(shader, "iTimeDelta")
|
||||
gl.glUniform1f(iTimeDelta_location, time_delta)
|
||||
iFrameRate_location = gl.glGetUniformLocation(shader, "iFrameRate")
|
||||
gl.glUniform1f(iFrameRate_location, frame_rate)
|
||||
iFrame_location = gl.glGetUniformLocation(shader, "iFrame")
|
||||
gl.glUniform1i(iFrame_location, frame)
|
||||
|
||||
def shadertoy_texture_update(texture, image, frame):
|
||||
if len(image.shape) == 4:
|
||||
image = image[frame]
|
||||
image = image.cpu().numpy()
|
||||
image = image[::-1, :, :]
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, texture)
|
||||
gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB, image.shape[1], image.shape[0], 0, gl.GL_RGB, gl.GL_FLOAT, image)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE)
|
||||
gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE)
|
||||
|
||||
def shadertoy_texture_bind(shader, textures):
|
||||
gl.glUseProgram(shader)
|
||||
for i in range(4):
|
||||
gl.glActiveTexture(gl.GL_TEXTURE0 + i) # type: ignore
|
||||
gl.glBindTexture(gl.GL_TEXTURE_2D, textures[i])
|
||||
iChannel_location = gl.glGetUniformLocation(shader, f"iChannel{i}")
|
||||
gl.glUniform1i(iChannel_location, i)
|
||||
|
||||
class FL_Shadertoy:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {"required": {"width": ("INT", {"default": 512, "min": 64, "max": 2048, "step": 8}),
|
||||
"height": ("INT", {"default": 512, "min": 64, "max": 2048, "step": 8}),
|
||||
"frame_count": ("INT", {"default": 1, "min": 1, "max": 262144}),
|
||||
"fps": ("INT", {"default": 1, "min": 1, "max": 120}),
|
||||
"source": (
|
||||
"STRING", {"default": SHADERTOY_DEFAULT, "multiline": True, "dynamicPrompts": False})},
|
||||
"optional": {"channel_0": ("IMAGE",),
|
||||
"channel_1": ("IMAGE",),
|
||||
"channel_2": ("IMAGE",),
|
||||
"channel_3": ("IMAGE",)}}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
FUNCTION = "render"
|
||||
|
||||
def render(self, width: int, height: int, frame_count: int, fps: int, source: str,
|
||||
channel_0: torch.Tensor | None = None, channel_1: torch.Tensor | None = None,
|
||||
channel_2: torch.Tensor | None = None, channel_3: torch.Tensor | None = None):
|
||||
fragment_source = SHADERTOY_HEADER
|
||||
fragment_source += source
|
||||
fragment_source += SHADERTOY_FOOTER
|
||||
|
||||
ctx, fbo, shader, textures = setup_render_resources(width, height, fragment_source)
|
||||
|
||||
images = []
|
||||
frame = 0
|
||||
total_frames = frame_count
|
||||
for _ in range(frame_count):
|
||||
shadertoy_vars_update(shader, width, height, frame * (1.0 / fps), (1.0 / fps), fps, frame)
|
||||
if channel_0 is not None: shadertoy_texture_update(textures[0], channel_0, frame)
|
||||
if channel_1 is not None: shadertoy_texture_update(textures[1], channel_1, frame)
|
||||
if channel_2 is not None: shadertoy_texture_update(textures[2], channel_2, frame)
|
||||
if channel_3 is not None: shadertoy_texture_update(textures[3], channel_3, frame)
|
||||
shadertoy_texture_bind(shader, textures)
|
||||
|
||||
image = render(width, height, fbo, shader)
|
||||
image = torch.from_numpy(image)[None,]
|
||||
images.append(image)
|
||||
|
||||
frame += 1
|
||||
|
||||
# Print progress update
|
||||
progress = frame / total_frames * 100
|
||||
sys.stdout.write(f"\rProcessing frames: {progress:.2f}%")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Print a new line after the progress update
|
||||
print()
|
||||
|
||||
render_resources_cleanup(ctx)
|
||||
|
||||
return (torch.cat(images, dim=0),)
|
||||
@@ -0,0 +1,41 @@
|
||||
import io
|
||||
import wave
|
||||
import numpy as np
|
||||
|
||||
class FL_AudioConverter:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"audio": ("AUDIO",),
|
||||
"sample_rate": ("INT", {"default": 44100, "description": "Sample rate of the audio"}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("VHS_AUDIO",)
|
||||
RETURN_NAMES = ("audio",)
|
||||
FUNCTION = "convert_audio"
|
||||
CATEGORY = "🏵️Fill Nodes"
|
||||
|
||||
def convert_audio(self, audio, sample_rate):
|
||||
# Extract the audio data and sample rate from the input tuple
|
||||
audio_data, _ = audio
|
||||
|
||||
# Convert the audio data to int16
|
||||
audio_data = (audio_data * 32767).astype(np.int16)
|
||||
|
||||
# Create a bytes buffer
|
||||
buffer = io.BytesIO()
|
||||
|
||||
# Write the audio data to the buffer using wave module
|
||||
with wave.open(buffer, 'wb') as wav_file:
|
||||
wav_file.setnchannels(1) # Mono audio
|
||||
wav_file.setsampwidth(2) # 2 bytes per sample (int16)
|
||||
wav_file.setframerate(sample_rate)
|
||||
wav_file.writeframes(audio_data.tobytes())
|
||||
|
||||
# Get the bytes from the buffer
|
||||
audio_bytes = buffer.getvalue()
|
||||
|
||||
# Return a lambda function that returns the audio bytes
|
||||
return (lambda: audio_bytes,)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Vendored
BIN
Binary file not shown.
Binary file not shown.
|
Before Width: | Height: | Size: 208 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 585 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 270 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 352 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 76 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 103 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 250 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 107 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 75 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 49 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 173 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 390 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 336 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 100 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 26 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 169 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 93 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 52 KiB |
@@ -1,10 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Update package lists
|
||||
apt-get update
|
||||
|
||||
# Install required system libraries for OpenGL
|
||||
apt-get install -y libgl1-mesa-glx freeglut3-dev
|
||||
|
||||
# Install Python dependencies
|
||||
pip install -r requirements.txt
|
||||
Vendored
BIN
Binary file not shown.
@@ -1,342 +0,0 @@
|
||||
# FL_Fal_GPTImage2_Edit: Fal AI GPT-Image-2 Edit endpoint
|
||||
# Takes N reference images + a single prompt, returns 1-4 edited images.
|
||||
import os
|
||||
import io
|
||||
import time
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
from typing import List, Optional
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
import fal_client
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
|
||||
IMAGE_SIZE_PRESETS = [
|
||||
"auto",
|
||||
"square_hd",
|
||||
"square",
|
||||
"portrait_4_3",
|
||||
"portrait_16_9",
|
||||
"landscape_4_3",
|
||||
"landscape_16_9",
|
||||
"custom",
|
||||
]
|
||||
QUALITY_OPTIONS = ["low", "medium", "high"]
|
||||
OUTPUT_FORMATS = ["png", "jpeg", "webp"]
|
||||
|
||||
|
||||
class FL_Fal_GPTImage2_Edit:
|
||||
"""
|
||||
ComfyUI node for fal.ai's `openai/gpt-image-2/edit` endpoint.
|
||||
|
||||
Takes a variable number of reference images (image_ref_1, image_ref_2, ...),
|
||||
uploads them to fal.media, then edits/composes against a shared prompt.
|
||||
Dynamic sockets are driven by the `image_count` widget + companion JS file.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("images", "image_urls", "status_msg")
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False, "default": ""}),
|
||||
"image_count": ("INT", {"default": 1, "min": 1, "max": 10, "step": 1}),
|
||||
"prompt": ("STRING", {"default": "A cozy scene with all these elements combined.",
|
||||
"multiline": True, "forceInput": False}),
|
||||
"image_size": (IMAGE_SIZE_PRESETS, {"default": "auto"}),
|
||||
"custom_width": ("INT", {"default": 1024, "min": 64, "max": 4096, "step": 8}),
|
||||
"custom_height": ("INT", {"default": 1024, "min": 64, "max": 4096, "step": 8}),
|
||||
"quality": (QUALITY_OPTIONS, {"default": "high"}),
|
||||
"num_images": ("INT", {"default": 1, "min": 1, "max": 4, "step": 1}),
|
||||
"output_format": (OUTPUT_FORMATS, {"default": "png"}),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}),
|
||||
"image_ref_1": ("IMAGE",),
|
||||
},
|
||||
"optional": {
|
||||
"mask": ("IMAGE",),
|
||||
"sync_mode": ("BOOLEAN", {"default": False}),
|
||||
"retry_indefinitely": ("BOOLEAN", {"default": False}),
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages: List[str] = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _log(self, message: str) -> str:
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted = f"[FL_Fal_GPTImage2_Edit] {timestamp}: {message}"
|
||||
print(formatted)
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
def _tensor_to_pil(tensor: Optional[torch.Tensor]) -> Optional[Image.Image]:
|
||||
if tensor is None or not isinstance(tensor, torch.Tensor):
|
||||
return None
|
||||
if tensor.ndim == 4:
|
||||
if tensor.shape[0] == 0:
|
||||
return None
|
||||
arr = tensor[0].cpu().numpy()
|
||||
elif tensor.ndim == 3:
|
||||
arr = tensor.cpu().numpy()
|
||||
else:
|
||||
return None
|
||||
arr = (arr * 255.0).clip(0, 255).astype(np.uint8)
|
||||
return Image.fromarray(arr)
|
||||
|
||||
def _create_error_image(self, error_message: str = "API Error",
|
||||
width: int = 1024, height: int = 1024) -> torch.Tensor:
|
||||
image = Image.new("RGB", (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
font = None
|
||||
for font_name in ("arial.ttf", "DejaVuSans.ttf", "FreeSans.ttf", "NotoSans-Regular.ttf"):
|
||||
try:
|
||||
font = ImageFont.truetype(font_name, 24)
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
if font is None:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
try:
|
||||
bbox = draw.textbbox((0, 0), error_message, font=font)
|
||||
tw, th = bbox[2] - bbox[0], bbox[3] - bbox[1]
|
||||
except AttributeError:
|
||||
tw, th = draw.textsize(error_message, font=font)
|
||||
draw.text(((width - tw) / 2, (height - th) / 2), error_message,
|
||||
fill=(255, 0, 0), font=font)
|
||||
arr = np.array(image).astype(np.float32) / 255.0
|
||||
return torch.from_numpy(arr).unsqueeze(0)
|
||||
|
||||
def _upload_pil(self, pil: Image.Image, label: str) -> Optional[str]:
|
||||
try:
|
||||
buf = io.BytesIO()
|
||||
pil.save(buf, format="PNG")
|
||||
url = fal_client.upload(buf.getvalue(), content_type="image/png")
|
||||
self._log(f"Uploaded {label} to CDN: {url[:80]}...")
|
||||
return url
|
||||
except Exception as e:
|
||||
self._log(f"Error uploading {label}: {e}")
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Async core
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _generate_async(
|
||||
self, api_key, prompt, ref_pils, mask_pil,
|
||||
image_size, custom_width, custom_height,
|
||||
quality, num_images, output_format,
|
||||
max_retries, retry_indefinitely, sync_mode,
|
||||
):
|
||||
os.environ["FAL_KEY"] = api_key.strip()
|
||||
|
||||
image_urls = []
|
||||
for idx, pil in enumerate(ref_pils, start=1):
|
||||
url = self._upload_pil(pil, f"image_ref_{idx}")
|
||||
if url is None:
|
||||
msg = f"Upload failed for image_ref_{idx}"
|
||||
return self._create_error_image(msg), "", msg
|
||||
image_urls.append(url)
|
||||
|
||||
mask_url = None
|
||||
if mask_pil is not None:
|
||||
mask_url = self._upload_pil(mask_pil, "mask")
|
||||
if mask_url is None:
|
||||
msg = "Upload failed for mask"
|
||||
return self._create_error_image(msg), "", msg
|
||||
|
||||
resolved_image_size: object
|
||||
if image_size == "custom":
|
||||
resolved_image_size = {"width": int(custom_width), "height": int(custom_height)}
|
||||
else:
|
||||
resolved_image_size = image_size
|
||||
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"image_urls": image_urls,
|
||||
"image_size": resolved_image_size,
|
||||
"quality": quality,
|
||||
"num_images": num_images,
|
||||
"output_format": output_format,
|
||||
"sync_mode": sync_mode,
|
||||
}
|
||||
if mask_url is not None:
|
||||
arguments["mask_url"] = mask_url
|
||||
|
||||
self._log(f"Calling openai/gpt-image-2/edit with {len(image_urls)} ref image(s), "
|
||||
f"quality={quality}, size={resolved_image_size}, n={num_images}")
|
||||
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for entry in update.logs:
|
||||
self._log(f"API Log: {entry.get('message', '')}")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
attempt = 0
|
||||
result = None
|
||||
last_err = ""
|
||||
while True:
|
||||
attempt += 1
|
||||
|
||||
def call():
|
||||
try:
|
||||
return fal_client.subscribe(
|
||||
"openai/gpt-image-2/edit",
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
except Exception as e:
|
||||
return {"__error__": str(e)}
|
||||
|
||||
raw = await loop.run_in_executor(None, call)
|
||||
if isinstance(raw, dict) and "__error__" in raw:
|
||||
last_err = raw["__error__"]
|
||||
self._log(f"Attempt {attempt} failed: {last_err}")
|
||||
if retry_indefinitely or attempt < max_retries:
|
||||
await asyncio.sleep(min(2 ** attempt, 30))
|
||||
continue
|
||||
msg = f"API error after {attempt} attempts: {last_err}"
|
||||
return self._create_error_image(msg), "", msg
|
||||
result = raw
|
||||
break
|
||||
|
||||
if not result or "images" not in result or not result["images"]:
|
||||
msg = "API returned no images"
|
||||
return self._create_error_image(msg), "", msg
|
||||
|
||||
out_urls = [img.get("url") for img in result["images"] if img.get("url")]
|
||||
if not out_urls:
|
||||
msg = "API response missing image URLs"
|
||||
return self._create_error_image(msg), "", msg
|
||||
|
||||
tensors = []
|
||||
for i, url in enumerate(out_urls):
|
||||
try:
|
||||
resp = requests.get(url, timeout=120)
|
||||
resp.raise_for_status()
|
||||
pil = Image.open(io.BytesIO(resp.content))
|
||||
if pil.mode != "RGB":
|
||||
pil = pil.convert("RGB")
|
||||
arr = np.array(pil).astype(np.float32) / 255.0
|
||||
tensors.append(torch.from_numpy(arr).unsqueeze(0))
|
||||
self._log(f"Downloaded result {i + 1}/{len(out_urls)} ({pil.size[0]}x{pil.size[1]})")
|
||||
except Exception as e:
|
||||
self._log(f"Failed to download {url}: {e}")
|
||||
|
||||
if not tensors:
|
||||
msg = "All result downloads failed"
|
||||
return self._create_error_image(msg), "", msg
|
||||
|
||||
combined = _pad_and_cat(tensors)
|
||||
joined_urls = " | ".join(out_urls)
|
||||
return combined, joined_urls, f"Success: {len(tensors)} image(s) generated"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def generate(self, api_key, image_count, prompt, image_size,
|
||||
custom_width, custom_height, quality, num_images, output_format,
|
||||
max_retries, image_ref_1,
|
||||
mask=None, sync_mode=False, retry_indefinitely=False, **kwargs):
|
||||
self.log_messages = []
|
||||
if not api_key:
|
||||
msg = "API key not provided."
|
||||
self._log(msg)
|
||||
return (self._create_error_image(msg), "", msg)
|
||||
|
||||
ref_tensors = [image_ref_1]
|
||||
for i in range(2, int(image_count) + 1):
|
||||
tensor = kwargs.get(f"image_ref_{i}")
|
||||
if tensor is not None:
|
||||
ref_tensors.append(tensor)
|
||||
|
||||
ref_pils: List[Image.Image] = []
|
||||
for i, t in enumerate(ref_tensors, start=1):
|
||||
pil = self._tensor_to_pil(t)
|
||||
if pil is None:
|
||||
self._log(f"image_ref_{i} is empty or invalid — skipping")
|
||||
continue
|
||||
ref_pils.append(pil)
|
||||
|
||||
if not ref_pils:
|
||||
msg = "At least one reference image is required."
|
||||
self._log(msg)
|
||||
return (self._create_error_image(msg), "", msg)
|
||||
|
||||
mask_pil = self._tensor_to_pil(mask) if mask is not None else None
|
||||
|
||||
pbar = ProgressBar(1)
|
||||
|
||||
async def runner():
|
||||
return await self._generate_async(
|
||||
api_key, prompt, ref_pils, mask_pil,
|
||||
image_size, custom_width, custom_height,
|
||||
quality, num_images, output_format,
|
||||
max_retries, retry_indefinitely, sync_mode,
|
||||
)
|
||||
|
||||
def run_sync():
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(runner())
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(run_sync)
|
||||
tensor, urls, status = future.result(timeout=600)
|
||||
except concurrent.futures.TimeoutError:
|
||||
msg = "Processing timed out after 10 minutes"
|
||||
self._log(msg)
|
||||
return (self._create_error_image(msg), "", msg)
|
||||
except Exception as e:
|
||||
msg = f"Processing error: {e}"
|
||||
self._log(msg)
|
||||
return (self._create_error_image(msg), "", msg)
|
||||
finally:
|
||||
pbar.update_absolute(1)
|
||||
|
||||
full_log = "Processing Logs:\n" + "\n".join(self.log_messages) + f"\n\nStatus: {status}"
|
||||
return (tensor, urls, full_log)
|
||||
|
||||
|
||||
def _pad_and_cat(tensors: List[torch.Tensor]) -> torch.Tensor:
|
||||
"""Pad a list of (1, H, W, C) tensors to the max HxW, then concat along batch dim."""
|
||||
if len(tensors) == 1:
|
||||
return tensors[0]
|
||||
max_h = max(t.shape[1] for t in tensors)
|
||||
max_w = max(t.shape[2] for t in tensors)
|
||||
out = []
|
||||
for t in tensors:
|
||||
h, w = t.shape[1], t.shape[2]
|
||||
if h == max_h and w == max_w:
|
||||
out.append(t)
|
||||
continue
|
||||
pad_h, pad_w = max_h - h, max_w - w
|
||||
pl, pr = pad_w // 2, pad_w - pad_w // 2
|
||||
pt, pb = pad_h // 2, pad_h - pad_h // 2
|
||||
padded = torch.nn.functional.pad(
|
||||
t.permute(0, 3, 1, 2),
|
||||
(pl, pr, pt, pb),
|
||||
mode="constant", value=0,
|
||||
).permute(0, 2, 3, 1)
|
||||
out.append(padded)
|
||||
return torch.cat(out, dim=0)
|
||||
@@ -1,424 +0,0 @@
|
||||
# FL_Fal_Gemini_ImageEdit: Fal AI Gemini 3 Pro Image Edit API Node
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import fal_client
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
|
||||
class FL_Fal_Gemini_ImageEdit:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Gemini 3 Pro Image Edit API.
|
||||
Takes up to 10 images and a prompt to edit them using Gemini's state-of-the-art multimodal capabilities.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "STRING", "STRING")
|
||||
RETURN_NAMES = ("images", "image_urls", "description", "status_msg")
|
||||
FUNCTION = "edit_images"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"prompt": ("STRING", {"default": "make a photo of the man driving the car down the california coastline",
|
||||
"multiline": True, "forceInput": True,
|
||||
"description": "The prompt for image editing (3-50,000 characters)"}),
|
||||
"num_images": ("INT", {"default": 1, "min": 1, "max": 4, "step": 1,
|
||||
"description": "Number of images to generate"}),
|
||||
"aspect_ratio": (["auto", "21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16"],
|
||||
{"default": "auto",
|
||||
"description": "Aspect ratio for output images"}),
|
||||
"resolution": (["1K", "2K", "4K"], {"default": "1K",
|
||||
"description": "Output resolution (4K costs 2x)"}),
|
||||
"output_format": (["jpeg", "png", "webp"], {"default": "png",
|
||||
"description": "Output image format"}),
|
||||
"sync_mode": ("BOOLEAN", {"default": False,
|
||||
"description": "When true, images returned as data URIs instead of URLs"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 666666,
|
||||
"description": "Random seed for reproducibility (0 = random)"}),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}),
|
||||
},
|
||||
"optional": {
|
||||
"image_1": ("IMAGE", {"description": "First input image to edit"}),
|
||||
"image_2": ("IMAGE", {"description": "Second input image to edit"}),
|
||||
"image_3": ("IMAGE", {"description": "Third input image to edit"}),
|
||||
"image_4": ("IMAGE", {"description": "Fourth input image to edit"}),
|
||||
"image_5": ("IMAGE", {"description": "Fifth input image to edit"}),
|
||||
"image_6": ("IMAGE", {"description": "Sixth input image to edit"}),
|
||||
"image_7": ("IMAGE", {"description": "Seventh input image to edit"}),
|
||||
"image_8": ("IMAGE", {"description": "Eighth input image to edit"}),
|
||||
"image_9": ("IMAGE", {"description": "Ninth input image to edit"}),
|
||||
"image_10": ("IMAGE", {"description": "Tenth input image to edit"}),
|
||||
"enable_web_search": ("BOOLEAN", {"default": False,
|
||||
"description": "Allow the model to use web search for latest information during generation"}),
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_Fal_Gemini_ImageEdit] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
if hasattr(self, 'log_messages'):
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _create_error_image(self, error_message="API Error", width=1024, height=1024):
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
font = None
|
||||
try:
|
||||
font_options = ['arial.ttf', 'DejaVuSans.ttf', 'FreeSans.ttf', 'NotoSans-Regular.ttf']
|
||||
for font_name in font_options:
|
||||
try:
|
||||
font = ImageFont.truetype(font_name, 24)
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
if font is None:
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Calculate text position (centered)
|
||||
try:
|
||||
text_bbox = draw.textbbox((0,0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
except AttributeError:
|
||||
text_width, text_height = draw.textsize(error_message, font=font)
|
||||
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log(f"Created error image: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _process_tensor_to_pil_list(self, tensor_image: Optional[torch.Tensor], image_name_prefix: str = "Image") -> Optional[List[Image.Image]]:
|
||||
if tensor_image is None:
|
||||
self._log(f"{image_name_prefix} input is None, skipping PIL conversion.")
|
||||
return None
|
||||
if not isinstance(tensor_image, torch.Tensor):
|
||||
self._log(f"{image_name_prefix} is not a tensor (type: {type(tensor_image)}), skipping.")
|
||||
return None
|
||||
|
||||
pil_images = []
|
||||
if tensor_image.ndim == 4: # Batch of images (B, H, W, C)
|
||||
if tensor_image.shape[0] == 0:
|
||||
self._log(f"{image_name_prefix} batch is empty (shape: {tensor_image.shape}).")
|
||||
return None
|
||||
for i in range(tensor_image.shape[0]):
|
||||
img_np = tensor_image[i].cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted {image_name_prefix} batch item {i} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
elif tensor_image.ndim == 3: # Single image (H, W, C)
|
||||
img_np = tensor_image.cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted single {image_name_prefix} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
else:
|
||||
self._log(f"Cannot convert {image_name_prefix} with ndim {tensor_image.ndim} (shape: {tensor_image.shape}) to PIL Image(s).")
|
||||
return None
|
||||
|
||||
return pil_images if pil_images else None
|
||||
|
||||
def _upload_image_to_fal(self, pil_image: Image.Image, api_key: str) -> str:
|
||||
"""Upload PIL image to fal.media CDN and return the URL.
|
||||
|
||||
This avoids the 10MB request body size limit by uploading images
|
||||
separately to fal's CDN instead of embedding base64 in the request.
|
||||
"""
|
||||
try:
|
||||
# Set the API key before upload
|
||||
os.environ["FAL_KEY"] = api_key.strip()
|
||||
|
||||
# Force reimport fal_client to pick up the new API key
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client as fresh_fal_client
|
||||
|
||||
# Save PIL image to bytes buffer
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
|
||||
# Upload to fal.media CDN using fal_client
|
||||
url = fresh_fal_client.upload(image_bytes, content_type="image/png")
|
||||
self._log(f"Uploaded image to fal CDN: {url[:80]}...")
|
||||
return url
|
||||
except Exception as e:
|
||||
self._log(f"Error uploading image to fal CDN: {str(e)}")
|
||||
raise
|
||||
|
||||
async def _edit_images_async(self, api_key, prompt, input_images, num_images, aspect_ratio, resolution, output_format, sync_mode, max_retries, enable_web_search=False, seed=0):
|
||||
try:
|
||||
self._log(f"Starting image editing with Gemini 3 Pro - prompt: '{prompt[:50]}...'")
|
||||
|
||||
# Set the API key FIRST - needed for both upload and API calls
|
||||
clean_api_key = api_key.strip()
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Upload images to fal.media CDN to avoid 10MB request body limit
|
||||
image_urls = []
|
||||
if input_images:
|
||||
self._log(f"Uploading {len(input_images)} images to fal.media CDN...")
|
||||
for i, pil_image in enumerate(input_images):
|
||||
try:
|
||||
img_url = self._upload_image_to_fal(pil_image, api_key)
|
||||
image_urls.append(img_url)
|
||||
self._log(f"Successfully uploaded image {i+1}/{len(input_images)} to CDN")
|
||||
except Exception as e:
|
||||
self._log(f"Error uploading image {i+1} to CDN: {str(e)}")
|
||||
error_msg = f"Error: Failed to upload image {i+1}: {str(e)}"
|
||||
return self._create_error_image(error_msg), "", "", error_msg
|
||||
else:
|
||||
error_msg = "Error: No images provided for editing"
|
||||
return self._create_error_image(error_msg), "", "", error_msg
|
||||
|
||||
# Prepare the arguments for fal_client
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"image_urls": image_urls,
|
||||
"num_images": num_images,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"output_format": output_format,
|
||||
"sync_mode": sync_mode,
|
||||
"limit_generations": False,
|
||||
"enable_web_search": enable_web_search
|
||||
}
|
||||
|
||||
# Add seed if provided (non-zero)
|
||||
if seed != 0:
|
||||
arguments["seed"] = seed
|
||||
self._log(f"Using seed: {seed}")
|
||||
|
||||
self._log(f"Calling Fal AI Gemini 3 Pro API with {len(image_urls)} images, {aspect_ratio} aspect ratio, {resolution} resolution...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
self._log(f"API Log: {log['message']}")
|
||||
|
||||
# Make the API call in executor to avoid blocking
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def make_fal_call():
|
||||
try:
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
result = fal_client.subscribe(
|
||||
"fal-ai/gemini-3-pro-image-preview/edit",
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
self._log(f"API call error: {str(e)}")
|
||||
return None
|
||||
|
||||
result = await loop.run_in_executor(None, make_fal_call)
|
||||
|
||||
if result is None:
|
||||
error_msg = "Error: API call failed"
|
||||
return self._create_error_image(error_msg), "", "", error_msg
|
||||
|
||||
self._log("API call completed successfully")
|
||||
|
||||
# Extract image URLs and description from the result
|
||||
output_image_urls = []
|
||||
description = ""
|
||||
|
||||
if "images" in result and len(result["images"]) > 0:
|
||||
for img_info in result["images"]:
|
||||
if "url" in img_info:
|
||||
output_image_urls.append(img_info["url"])
|
||||
self._log(f"Found {len(output_image_urls)} edited images in response")
|
||||
else:
|
||||
self._log("Warning: No images found in result")
|
||||
error_msg = "Error: No images in API response"
|
||||
return self._create_error_image(error_msg), "", "", error_msg
|
||||
|
||||
# Extract description
|
||||
if "description" in result:
|
||||
description = result["description"]
|
||||
self._log(f"Received description: {description[:100]}...")
|
||||
else:
|
||||
description = "No description provided"
|
||||
|
||||
# Download and process all generated images
|
||||
try:
|
||||
self._log(f"Downloading {len(output_image_urls)} edited images...")
|
||||
|
||||
processed_images = []
|
||||
url_list = []
|
||||
|
||||
for i, image_url in enumerate(output_image_urls):
|
||||
# Download image
|
||||
dl_response = requests.get(image_url)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Convert to PIL Image
|
||||
pil_image = Image.open(io.BytesIO(dl_response.content))
|
||||
|
||||
# Convert to RGB if necessary
|
||||
if pil_image.mode != 'RGB':
|
||||
pil_image = pil_image.convert('RGB')
|
||||
|
||||
# Convert to numpy array
|
||||
np_image = np.array(pil_image).astype(np.float32) / 255.0
|
||||
|
||||
# Convert to tensor (add batch dimension)
|
||||
image_tensor = torch.from_numpy(np_image).unsqueeze(0)
|
||||
processed_images.append(image_tensor)
|
||||
url_list.append(image_url)
|
||||
|
||||
self._log(f"Processed image {i+1}/{len(output_image_urls)} with shape {image_tensor.shape}")
|
||||
|
||||
# Concatenate all images
|
||||
if len(processed_images) > 1:
|
||||
# Handle multiple images - ensure same dimensions
|
||||
max_height = max(img.shape[1] for img in processed_images)
|
||||
max_width = max(img.shape[2] for img in processed_images)
|
||||
|
||||
resized_images = []
|
||||
for img_tensor in processed_images:
|
||||
current_h, current_w = img_tensor.shape[1], img_tensor.shape[2]
|
||||
|
||||
if current_h == max_height and current_w == max_width:
|
||||
resized_images.append(img_tensor)
|
||||
else:
|
||||
# Pad the image
|
||||
pad_h = max_height - current_h
|
||||
pad_w = max_width - current_w
|
||||
pad_left = pad_w // 2
|
||||
pad_right = pad_w - pad_left
|
||||
pad_top = pad_h // 2
|
||||
pad_bottom = pad_h - pad_top
|
||||
|
||||
padded_img = torch.nn.functional.pad(
|
||||
img_tensor.permute(0, 3, 1, 2),
|
||||
(pad_left, pad_right, pad_top, pad_bottom),
|
||||
mode='constant',
|
||||
value=0
|
||||
).permute(0, 2, 3, 1)
|
||||
|
||||
resized_images.append(padded_img)
|
||||
|
||||
combined_tensor = torch.cat(resized_images, dim=0)
|
||||
else:
|
||||
combined_tensor = processed_images[0]
|
||||
|
||||
combined_urls = " | ".join(url_list)
|
||||
|
||||
self._log(f"Successfully processed {len(processed_images)} images with final shape {combined_tensor.shape}")
|
||||
|
||||
return combined_tensor, combined_urls, description, f"Success: {len(processed_images)} images edited successfully"
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Download Error: {str(e)}"
|
||||
self._log(error_msg)
|
||||
return self._create_error_image(error_msg), "", "", error_msg
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error in async image editing: {str(e)}")
|
||||
error_msg = f"Error: {str(e)}"
|
||||
return self._create_error_image(error_msg), "", "", error_msg
|
||||
|
||||
def edit_images(self, api_key, prompt, num_images=1, aspect_ratio="auto", resolution="1K", output_format="png",
|
||||
sync_mode=False, max_retries=3, seed=0, image_1=None, image_2=None, image_3=None, image_4=None,
|
||||
image_5=None, image_6=None, image_7=None, image_8=None, image_9=None, image_10=None,
|
||||
enable_web_search=False, **kwargs):
|
||||
self.log_messages = []
|
||||
if not api_key:
|
||||
error_msg = "API key not provided."
|
||||
self._log(error_msg)
|
||||
error_img_instance = self._create_error_image(error_msg)
|
||||
return (error_img_instance, "", "", error_msg)
|
||||
|
||||
# Collect all input images
|
||||
input_images = []
|
||||
input_tensors = [image_1, image_2, image_3, image_4, image_5, image_6, image_7, image_8, image_9, image_10]
|
||||
|
||||
for i, tensor in enumerate(input_tensors):
|
||||
if tensor is not None:
|
||||
pil_images = self._process_tensor_to_pil_list(tensor, f"Image{i+1}")
|
||||
if pil_images:
|
||||
input_images.extend(pil_images)
|
||||
|
||||
if not input_images:
|
||||
error_msg = "No valid input images provided."
|
||||
self._log(error_msg)
|
||||
error_img_instance = self._create_error_image(error_msg)
|
||||
return (error_img_instance, "", "", error_msg)
|
||||
|
||||
self._log(f"Processing {len(input_images)} input images for editing")
|
||||
|
||||
# Run async processing using thread pool to avoid event loop conflicts
|
||||
def run_sync_edit():
|
||||
"""Run async edit in a new thread with its own event loop"""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(self._edit_images_async(
|
||||
api_key, prompt, input_images, num_images, aspect_ratio, resolution,
|
||||
output_format, sync_mode, max_retries, enable_web_search, seed
|
||||
))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
result = None
|
||||
try:
|
||||
# Use thread pool executor to run async code in separate thread
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(run_sync_edit)
|
||||
result = future.result(timeout=300) # 5 minute timeout
|
||||
except concurrent.futures.TimeoutError:
|
||||
self._log("Processing timed out after 5 minutes")
|
||||
error_img = self._create_error_image("Processing timeout")
|
||||
return (error_img, "", "", "Processing timed out after 5 minutes")
|
||||
except Exception as e:
|
||||
self._log(f"Error in processing: {str(e)}")
|
||||
error_img = self._create_error_image(f"Processing error: {str(e)}")
|
||||
return (error_img, "", "", f"Processing error: {str(e)}")
|
||||
|
||||
if result is None:
|
||||
error_img = self._create_error_image("Processing failed to produce results")
|
||||
return (error_img, "", "", "Processing failed to produce results")
|
||||
|
||||
# Extract results
|
||||
images, image_urls, description, status_msg = result
|
||||
|
||||
# Combine log messages with status
|
||||
final_log_output = "Processing Logs:\n" + "\n".join(self.log_messages) + "\n\n" + status_msg
|
||||
|
||||
return (images, image_urls, description, final_log_output)
|
||||
@@ -1,353 +0,0 @@
|
||||
# FL_Fal_Kling_AIAvatar: Fal AI Kling Video AI Avatar API Node
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import tempfile
|
||||
import cv2
|
||||
import base64
|
||||
import fal_client
|
||||
import torchaudio
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class FL_Fal_Kling_AIAvatar:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Kling Video AI Avatar API.
|
||||
Takes an image and audio to generate realistic avatar videos.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames", "audio", "video_url", "status_msg")
|
||||
FUNCTION = "generate_ai_avatar"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"image": ("IMAGE", {"description": "Input image to use as avatar"}),
|
||||
"audio": ("AUDIO", {"description": "Input audio tensor"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 999999,
|
||||
"description": "Random seed (0 = random, max 6 digits)"}),
|
||||
"nth_frame": ("INT", {"default": 1, "min": 1, "max": 4,
|
||||
"description": "Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)"})
|
||||
},
|
||||
"optional": {
|
||||
"prompt": ("STRING", {"default": "", "multiline": True,
|
||||
"description": "Optional text prompt to refine video generation"})
|
||||
}
|
||||
}
|
||||
|
||||
def sanitize_error_message(self, msg):
|
||||
"""Remove potentially large base64 data from error messages"""
|
||||
if not isinstance(msg, str):
|
||||
msg = str(msg)
|
||||
# Replace data URIs with placeholders to avoid wall of text
|
||||
import re
|
||||
msg = re.sub(r'data:[^;]+;base64,[A-Za-z0-9+/=]{100,}', '<data_uri_removed>', msg)
|
||||
return msg
|
||||
|
||||
def generate_ai_avatar(self, api_key, image, audio, seed=0, nth_frame=1, prompt=""):
|
||||
"""
|
||||
Generate AI Avatar video using Fal AI Kling Video AI Avatar API
|
||||
|
||||
Args:
|
||||
api_key: Fal AI API key
|
||||
image: Input image tensor to use as avatar
|
||||
audio: Input audio tensor
|
||||
seed: Random seed for generation (0 = random, max 6 digits)
|
||||
nth_frame: Extract every Nth frame
|
||||
prompt: Optional text prompt to refine video generation
|
||||
|
||||
Returns:
|
||||
Tuple of (frames_tensor, audio, video_url, status_message)
|
||||
"""
|
||||
try:
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
empty_audio = {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
clean_msg = self.sanitize_error_message(error_msg)
|
||||
return empty_tensor, empty_audio, "", clean_msg
|
||||
|
||||
# Clear any existing FAL_KEY environment variable to prevent caching issues
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
print("[Fal Kling AI Avatar] Cleared existing FAL_KEY environment variable")
|
||||
|
||||
# 1. Validate API key
|
||||
if not api_key or api_key.strip() == "":
|
||||
return error_return("Error: API Key is required")
|
||||
|
||||
# 2. Validate required inputs
|
||||
if image is None:
|
||||
return error_return("Error: Image input is required")
|
||||
|
||||
if audio is None:
|
||||
return error_return("Error: Audio input is required")
|
||||
|
||||
# 3. Process seed (ensure it's within 6-digit limit)
|
||||
if seed < 0 or seed > 999999:
|
||||
return error_return("Error: Seed must be between 0 and 999999 (6 digits max)")
|
||||
|
||||
# 4. Set up tensors for processing after API key setup
|
||||
audio_tensor_to_process = audio
|
||||
image_tensor_to_process = image
|
||||
print(f"[Fal Kling AI Avatar] Image and audio tensors provided, will process after API setup")
|
||||
print(f"[Fal Kling AI Avatar] Using seed: {seed}")
|
||||
|
||||
print(f"[Fal Kling AI Avatar] Starting AI Avatar generation...")
|
||||
|
||||
# Prepare the API request
|
||||
clean_api_key = api_key.strip()
|
||||
|
||||
# Prepare the arguments for fal_client (URLs will be set after upload)
|
||||
arguments = {
|
||||
"image_url": "", # Will be set after image upload
|
||||
"audio_url": "" # Will be set after audio upload
|
||||
}
|
||||
|
||||
# Add seed if specified (0 means random)
|
||||
if seed > 0:
|
||||
arguments["seed"] = seed
|
||||
|
||||
# Add optional prompt if provided
|
||||
if prompt and prompt.strip():
|
||||
arguments["prompt"] = prompt.strip()
|
||||
print(f"[Fal Kling AI Avatar] Prompt: {prompt[:100]}{'...' if len(prompt) > 100 else ''}")
|
||||
|
||||
# Remove any None values from arguments
|
||||
arguments = {k: v for k, v in arguments.items() if v is not None and v != ""}
|
||||
|
||||
# Print arguments without exposing potentially large base64 data
|
||||
safe_arguments = {k: v if not (isinstance(v, str) and v.startswith('data:')) else f"<data_uri_{len(v)}_chars>" for k, v in arguments.items()}
|
||||
print(f"[Fal Kling AI Avatar] API arguments: {safe_arguments}")
|
||||
|
||||
# Set the API key as an environment variable for fal_client
|
||||
key_preview = clean_api_key[:8] + "..." if len(clean_api_key) > 8 else "invalid_key"
|
||||
print(f"[Fal Kling AI Avatar] Using API key starting with: {key_preview}")
|
||||
|
||||
# Clear and set the environment variable
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
print(f"[Fal Kling AI Avatar] Calling Fal AI API with fal_client...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
print(f"[Fal Kling AI Avatar] Log: {log['message']}")
|
||||
|
||||
try:
|
||||
# Use the Kling AI Avatar endpoint
|
||||
endpoint = "fal-ai/kling-video/v1/pro/ai-avatar"
|
||||
print(f"[Fal Kling AI Avatar] Using endpoint: {endpoint}")
|
||||
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Process audio tensor upload if needed (after fal_client is properly loaded)
|
||||
if audio_tensor_to_process is not None:
|
||||
try:
|
||||
# Convert audio tensor to temporary file and upload to Fal
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_audio:
|
||||
temp_audio_path = temp_audio.name
|
||||
|
||||
# Save audio tensor to temporary file
|
||||
waveform = audio_tensor_to_process['waveform']
|
||||
sample_rate = audio_tensor_to_process['sample_rate']
|
||||
|
||||
# Ensure waveform is in correct format for torchaudio.save
|
||||
if len(waveform.shape) == 3: # [batch, channels, samples]
|
||||
waveform = waveform.squeeze(0) # Remove batch dimension
|
||||
|
||||
torchaudio.save(temp_audio_path, waveform, sample_rate)
|
||||
print(f"[Fal Kling AI Avatar] Saved audio tensor to temporary file: {temp_audio_path}")
|
||||
|
||||
# Upload to Fal
|
||||
final_audio_url = fal_client.upload_file(temp_audio_path)
|
||||
print(f"[Fal Kling AI Avatar] Uploaded audio to Fal: {final_audio_url}")
|
||||
|
||||
# Update arguments with the uploaded audio URL
|
||||
arguments["audio_url"] = final_audio_url
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_audio_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Kling AI Avatar] Error processing audio tensor: {str(e)}")
|
||||
return error_return(f"Error: Failed to process audio: {str(e)}")
|
||||
|
||||
# Process image tensor upload if needed (after fal_client is properly loaded)
|
||||
if image_tensor_to_process is not None:
|
||||
try:
|
||||
# Convert image tensor to temporary file and upload to Fal
|
||||
with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as temp_image:
|
||||
temp_image_path = temp_image.name
|
||||
|
||||
# Convert image tensor to PIL Image
|
||||
if len(image_tensor_to_process.shape) == 4:
|
||||
image_tensor = image_tensor_to_process[0]
|
||||
else:
|
||||
image_tensor = image_tensor_to_process
|
||||
|
||||
# Convert to uint8
|
||||
if image_tensor.dtype != torch.uint8:
|
||||
image_tensor = (image_tensor * 255).to(torch.uint8)
|
||||
|
||||
# Convert to numpy for PIL
|
||||
np_img = image_tensor.cpu().numpy()
|
||||
pil_image = Image.fromarray(np_img)
|
||||
|
||||
# Save to temporary file
|
||||
pil_image.save(temp_image_path, format="PNG")
|
||||
print(f"[Fal Kling AI Avatar] Saved image tensor to temporary file: {temp_image_path}")
|
||||
|
||||
# Upload to Fal
|
||||
final_image_url = fal_client.upload_file(temp_image_path)
|
||||
print(f"[Fal Kling AI Avatar] Uploaded image to Fal: {final_image_url}")
|
||||
|
||||
# Update arguments with the uploaded image URL
|
||||
arguments["image_url"] = final_image_url
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_image_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Kling AI Avatar] Error processing image tensor: {str(e)}")
|
||||
return error_return(f"Error: Failed to process image: {str(e)}")
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
print(f"[Fal Kling AI Avatar] Making API call with fal_client.subscribe...")
|
||||
result = fal_client.subscribe(
|
||||
endpoint,
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
|
||||
print(f"[Fal Kling AI Avatar] API call completed successfully")
|
||||
except Exception as e:
|
||||
error_msg = f"API Error: {str(e)}"
|
||||
print(f"[Fal Kling AI Avatar] {error_msg}")
|
||||
return error_return(error_msg)
|
||||
|
||||
# Extract video URL from the result
|
||||
if "video" in result and "url" in result["video"]:
|
||||
output_video_url = result["video"]["url"]
|
||||
print(f"[Fal Kling AI Avatar] Video ready! URL: {output_video_url}")
|
||||
|
||||
# Download and process the video
|
||||
try:
|
||||
print(f"[Fal Kling AI Avatar] Downloading video...")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
# Download video to temp file
|
||||
dl_response = requests.get(output_video_url, stream=True)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Get file size for progress bar
|
||||
file_size = int(dl_response.headers.get('content-length', 0))
|
||||
progress_bar = tqdm(total=file_size, unit='B', unit_scale=True, desc="Downloading AI Avatar Video")
|
||||
|
||||
for chunk in dl_response.iter_content(chunk_size=8192):
|
||||
temp_video.write(chunk)
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Extract frames using OpenCV
|
||||
print(f"[Fal Kling AI Avatar] Extracting frames from video...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
os.unlink(temp_video_path) # Clean up temp file
|
||||
return error_return("Could not open video file")
|
||||
|
||||
# Get video properties
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
print(f"[Fal Kling AI Avatar] Video has {total_frames} frames at {fps} FPS")
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
# Use nth_frame directly as the stride
|
||||
stride = nth_frame
|
||||
|
||||
# Calculate approximately how many frames we'll extract
|
||||
frames_to_extract = total_frames // stride + (1 if total_frames % stride > 0 else 0)
|
||||
|
||||
progress_bar = tqdm(total=frames_to_extract, desc="Extracting frames")
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % stride == 0 and len(frames) < frames_to_extract:
|
||||
# Convert BGR to RGB (OpenCV uses BGR by default)
|
||||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Normalize to 0-1 range for ComfyUI
|
||||
normalized_frame = rgb_frame.astype(np.float32) / 255.0
|
||||
|
||||
frames.append(normalized_frame)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Break if we've extracted enough frames
|
||||
if len(frames) >= frames_to_extract:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
|
||||
progress_bar.close()
|
||||
cap.release()
|
||||
|
||||
# Clean up temp file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[Fal Kling AI Avatar] Extracted {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
|
||||
# Prepare audio output - use the original input audio if provided
|
||||
output_audio = audio if audio is not None else {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
|
||||
return frames_tensor, output_audio, output_video_url, "Success: AI Avatar video generated and processed"
|
||||
else:
|
||||
return error_return("No frames could be extracted")
|
||||
|
||||
except Exception as e:
|
||||
return error_return(f"Processing Error: {str(e)}")
|
||||
else:
|
||||
return error_return("No video URL in API response")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Kling AI Avatar] Error: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
empty_audio = {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
# Sanitize error message to remove potential base64 data
|
||||
clean_error = self.sanitize_error_message(f"Error: {str(e)}")
|
||||
return empty_tensor, empty_audio, "", clean_error
|
||||
@@ -1,456 +0,0 @@
|
||||
# FL_Fal_Kontext: Fal AI Flux Pro Kontext API Node with async support
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import fal_client
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import random
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
|
||||
class FL_Fal_Kontext:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Flux Pro Kontext API.
|
||||
Takes multiple image/prompt pairs and generates new images using Fal AI's flux-pro/kontext endpoint.
|
||||
Supports async processing for multiple inputs.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("images", "image_urls", "status_msg")
|
||||
FUNCTION = "generate_images_advanced"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"inputcount": ("INT", {"default": 1, "min": 1, "max": 100, "step": 1}),
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647,
|
||||
"description": "Random seed for image generation (0 = random)"}),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}),
|
||||
"prompt_1": ("STRING", {"default": "Put a donut next to the flour.",
|
||||
"multiline": True, "forceInput": True,
|
||||
"description": "Text prompt describing what to add or modify in the image"}),
|
||||
"guidance_scale": ("FLOAT", {"default": 3.5, "min": 1.0, "max": 20.0, "step": 0.1,
|
||||
"description": "CFG scale - how closely to follow the prompt"}),
|
||||
"num_images": ("INT", {"default": 1, "min": 1, "max": 4, "step": 1,
|
||||
"description": "Number of images to generate per input"}),
|
||||
"aspect_ratio": (["21:9", "16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16", "9:21"],
|
||||
{"default": "1:1", "description": "Aspect ratio of generated images"}),
|
||||
"output_format": (["jpeg", "png"], {"default": "jpeg",
|
||||
"description": "Output image format"}),
|
||||
"safety_tolerance": (["1", "2", "3", "4", "5", "6"], {"default": "2",
|
||||
"description": "Safety tolerance (1=strict, 6=permissive)"}),
|
||||
},
|
||||
"optional": {
|
||||
"image_1": ("IMAGE", {"description": "Input image to modify"}),
|
||||
"retry_indefinitely": ("BOOLEAN", {"default": False}),
|
||||
"sync_mode": ("BOOLEAN", {"default": False,
|
||||
"description": "Wait for image generation before returning (higher latency)"}),
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_Fal_Kontext] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
if hasattr(self, 'log_messages'):
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _create_error_image(self, error_message="API Error", width=1024, height=1024):
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
font = None
|
||||
try:
|
||||
font_options = ['arial.ttf', 'DejaVuSans.ttf', 'FreeSans.ttf', 'NotoSans-Regular.ttf']
|
||||
for font_name in font_options:
|
||||
try:
|
||||
font = ImageFont.truetype(font_name, 24)
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
if font is None:
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Calculate text position (centered)
|
||||
try:
|
||||
text_bbox = draw.textbbox((0,0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
except AttributeError:
|
||||
text_width, text_height = draw.textsize(error_message, font=font)
|
||||
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log(f"Created error image: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _process_tensor_to_pil_list(self, tensor_image: Optional[torch.Tensor], image_name_prefix: str = "Image") -> Optional[List[Image.Image]]:
|
||||
if tensor_image is None:
|
||||
self._log(f"{image_name_prefix} input is None, skipping PIL conversion.")
|
||||
return None
|
||||
if not isinstance(tensor_image, torch.Tensor):
|
||||
self._log(f"{image_name_prefix} is not a tensor (type: {type(tensor_image)}), skipping.")
|
||||
return None
|
||||
|
||||
pil_images = []
|
||||
if tensor_image.ndim == 4: # Batch of images (B, H, W, C)
|
||||
if tensor_image.shape[0] == 0:
|
||||
self._log(f"{image_name_prefix} batch is empty (shape: {tensor_image.shape}).")
|
||||
return None
|
||||
for i in range(tensor_image.shape[0]):
|
||||
img_np = tensor_image[i].cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted {image_name_prefix} batch item {i} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
elif tensor_image.ndim == 3: # Single image (H, W, C)
|
||||
img_np = tensor_image.cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted single {image_name_prefix} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
else:
|
||||
self._log(f"Cannot convert {image_name_prefix} with ndim {tensor_image.ndim} (shape: {tensor_image.shape}) to PIL Image(s).")
|
||||
return None
|
||||
|
||||
return pil_images if pil_images else None
|
||||
|
||||
async def _generate_single_image_async(self, api_key, prompt_text, input_pil_images: Optional[List[Image.Image]], seed_val, max_retries, retry_indefinitely, guidance_scale, num_images, aspect_ratio, output_format, safety_tolerance, sync_mode, call_id):
|
||||
try:
|
||||
# Calculate seed
|
||||
actual_seed = seed_val if seed_val != 0 else random.randint(1, 2147483647)
|
||||
|
||||
self._log(f"[Call {call_id}] Generating image with seed {actual_seed} for prompt: '{prompt_text[:50]}...'")
|
||||
|
||||
# Set API key FIRST - needed for CDN upload
|
||||
clean_api_key = api_key.strip()
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Upload image to fal.media CDN
|
||||
img_url = None
|
||||
if input_pil_images and len(input_pil_images) > 0:
|
||||
pil_image = input_pil_images[0] # Take first image
|
||||
try:
|
||||
# Upload PIL image to fal.media CDN
|
||||
self._log(f"[Call {call_id}] Uploading image to fal.media CDN...")
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
img_url = fal_client.upload(image_bytes, content_type="image/png")
|
||||
self._log(f"[Call {call_id}] Uploaded image to CDN: {img_url[:80]}...")
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] Error uploading image to CDN: {str(e)}")
|
||||
error_msg = f"Call {call_id} Error: Failed to upload image: {str(e)}"
|
||||
return self._create_error_image(error_msg), "", error_msg, call_id
|
||||
else:
|
||||
error_msg = f"Call {call_id} Error: No image provided"
|
||||
return self._create_error_image(error_msg), "", error_msg, call_id
|
||||
|
||||
# Prepare the arguments for fal_client
|
||||
arguments = {
|
||||
"prompt": prompt_text,
|
||||
"image_url": img_url,
|
||||
"seed": actual_seed,
|
||||
"guidance_scale": guidance_scale,
|
||||
"num_images": num_images,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"output_format": output_format,
|
||||
"safety_tolerance": safety_tolerance,
|
||||
"sync_mode": sync_mode
|
||||
}
|
||||
|
||||
self._log(f"[Call {call_id}] Calling Fal AI API with fal_client...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
self._log(f"[Call {call_id}] API Log: {log['message']}")
|
||||
|
||||
# Make the API call in executor to avoid blocking
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def make_fal_call():
|
||||
try:
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
result = fal_client.subscribe(
|
||||
"fal-ai/flux-pro/kontext",
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] API call error: {str(e)}")
|
||||
return None
|
||||
|
||||
result = await loop.run_in_executor(None, make_fal_call)
|
||||
|
||||
if result is None:
|
||||
error_msg = f"Call {call_id} Error: API call failed"
|
||||
return self._create_error_image(error_msg), "", error_msg, call_id
|
||||
|
||||
self._log(f"[Call {call_id}] API call completed successfully")
|
||||
|
||||
# Extract image URLs from the result
|
||||
image_urls = []
|
||||
if "images" in result and len(result["images"]) > 0:
|
||||
for img_info in result["images"]:
|
||||
if "url" in img_info:
|
||||
image_urls.append(img_info["url"])
|
||||
self._log(f"[Call {call_id}] Found {len(image_urls)} images in response")
|
||||
elif "image" in result and "url" in result["image"]:
|
||||
image_urls.append(result["image"]["url"])
|
||||
self._log(f"[Call {call_id}] Found 1 image in response (legacy format)")
|
||||
else:
|
||||
self._log(f"[Call {call_id}] Warning: No image URLs found in result")
|
||||
error_msg = f"Call {call_id} Error: No image URLs in API response"
|
||||
return self._create_error_image(error_msg), "", error_msg, call_id
|
||||
|
||||
# Download and process all generated images
|
||||
try:
|
||||
self._log(f"[Call {call_id}] Downloading {len(image_urls)} generated images...")
|
||||
|
||||
processed_images = []
|
||||
url_list = []
|
||||
|
||||
for i, image_url in enumerate(image_urls):
|
||||
# Download image
|
||||
dl_response = requests.get(image_url)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Convert to PIL Image
|
||||
pil_image = Image.open(io.BytesIO(dl_response.content))
|
||||
|
||||
# Convert to RGB if necessary
|
||||
if pil_image.mode != 'RGB':
|
||||
pil_image = pil_image.convert('RGB')
|
||||
|
||||
# Convert to numpy array
|
||||
np_image = np.array(pil_image).astype(np.float32) / 255.0
|
||||
|
||||
# Convert to tensor (add batch dimension)
|
||||
image_tensor = torch.from_numpy(np_image).unsqueeze(0)
|
||||
processed_images.append(image_tensor)
|
||||
url_list.append(image_url)
|
||||
|
||||
self._log(f"[Call {call_id}] Processed image {i+1}/{len(image_urls)} with shape {image_tensor.shape}")
|
||||
|
||||
# Concatenate all images from this API call
|
||||
if len(processed_images) > 1:
|
||||
# Handle multiple images - need to ensure same dimensions
|
||||
max_height = max(img.shape[1] for img in processed_images)
|
||||
max_width = max(img.shape[2] for img in processed_images)
|
||||
|
||||
resized_images = []
|
||||
for img_tensor in processed_images:
|
||||
current_h, current_w = img_tensor.shape[1], img_tensor.shape[2]
|
||||
|
||||
if current_h == max_height and current_w == max_width:
|
||||
resized_images.append(img_tensor)
|
||||
else:
|
||||
# Pad the image
|
||||
pad_h = max_height - current_h
|
||||
pad_w = max_width - current_w
|
||||
pad_left = pad_w // 2
|
||||
pad_right = pad_w - pad_left
|
||||
pad_top = pad_h // 2
|
||||
pad_bottom = pad_h - pad_top
|
||||
|
||||
padded_img = torch.nn.functional.pad(
|
||||
img_tensor.permute(0, 3, 1, 2),
|
||||
(pad_left, pad_right, pad_top, pad_bottom),
|
||||
mode='constant',
|
||||
value=0
|
||||
).permute(0, 2, 3, 1)
|
||||
|
||||
resized_images.append(padded_img)
|
||||
|
||||
combined_tensor = torch.cat(resized_images, dim=0)
|
||||
else:
|
||||
combined_tensor = processed_images[0]
|
||||
|
||||
combined_urls = " | ".join(url_list)
|
||||
|
||||
self._log(f"[Call {call_id}] Successfully processed {len(processed_images)} images with final shape {combined_tensor.shape}")
|
||||
|
||||
return combined_tensor, combined_urls, f"Call {call_id} Success: {len(processed_images)} images generated successfully", call_id
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Call {call_id} Download Error: {str(e)}"
|
||||
self._log(error_msg)
|
||||
return self._create_error_image(error_msg), "", error_msg, call_id
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] Error in async generation: {str(e)}")
|
||||
error_msg = f"Call {call_id} Error: {str(e)}"
|
||||
return self._create_error_image(error_msg), "", error_msg, call_id
|
||||
|
||||
def generate_images_advanced(self, inputcount, api_key, seed=0, max_retries=3, prompt_1="Put a donut next to the flour.", guidance_scale=3.5, num_images=1, aspect_ratio="1:1", output_format="jpeg", safety_tolerance="2", image_1=None, retry_indefinitely=False, sync_mode=False, **kwargs):
|
||||
self.log_messages = []
|
||||
if not api_key:
|
||||
error_msg = "API key not provided."
|
||||
self._log(error_msg)
|
||||
error_img_instance = self._create_error_image(error_msg)
|
||||
return ([error_img_instance] * inputcount, "", error_msg)
|
||||
|
||||
pbar = ProgressBar(inputcount) # Initialize progress bar
|
||||
|
||||
# Setup async tasks for each input
|
||||
async def run_batch():
|
||||
tasks = []
|
||||
|
||||
for slot_idx in range(1, inputcount + 1):
|
||||
current_prompt = prompt_1 if slot_idx == 1 else kwargs.get(f"prompt_{slot_idx}", f"Default prompt for image {slot_idx}")
|
||||
|
||||
current_image_tensor_for_slot = None
|
||||
if slot_idx == 1:
|
||||
current_image_tensor_for_slot = image_1
|
||||
else:
|
||||
current_image_tensor_for_slot = kwargs.get(f"image_{slot_idx}")
|
||||
|
||||
pil_images_for_this_slot = self._process_tensor_to_pil_list(current_image_tensor_for_slot, f"InputSlot{slot_idx}")
|
||||
|
||||
current_task_seed = seed + (slot_idx - 1) if seed != 0 else 0
|
||||
task_call_id = str(slot_idx)
|
||||
|
||||
tasks.append(self._generate_single_image_async(
|
||||
api_key, current_prompt, pil_images_for_this_slot,
|
||||
current_task_seed, max_retries, retry_indefinitely,
|
||||
guidance_scale, num_images, aspect_ratio, output_format,
|
||||
safety_tolerance, sync_mode, task_call_id
|
||||
))
|
||||
pbar.update_absolute(slot_idx) # Update progress bar after task is added
|
||||
|
||||
if not tasks:
|
||||
self._log("No tasks were created. This might indicate an issue with inputcount or logic.")
|
||||
return []
|
||||
|
||||
# Run all tasks concurrently
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
# Run the async batch processing using thread pool to avoid event loop conflicts
|
||||
def run_sync_batch():
|
||||
"""Run async batch in a new thread with its own event loop"""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(run_batch())
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
results_with_id = None # Initialize results
|
||||
try:
|
||||
# Use thread pool executor to run async code in separate thread
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(run_sync_batch)
|
||||
results_with_id = future.result(timeout=300) # 5 minute timeout
|
||||
except concurrent.futures.TimeoutError:
|
||||
self._log("Async processing timed out after 5 minutes")
|
||||
error_imgs = [self._create_error_image("Processing timeout")] * inputcount
|
||||
return (error_imgs, "", "Processing timed out after 5 minutes")
|
||||
except Exception as e:
|
||||
self._log(f"Error in async processing: {str(e)}")
|
||||
# Create batch of error images
|
||||
error_imgs = [self._create_error_image(f"Async processing error: {str(e)}")] * inputcount
|
||||
return (error_imgs, "", f"Async processing error: {str(e)}")
|
||||
|
||||
# Process results (ensure results is not None if an error occurred before assignment)
|
||||
if results_with_id is None:
|
||||
self._log("Async processing did not yield results, possibly due to an earlier error before gather.")
|
||||
error_imgs = [self._create_error_image("Async processing failed to produce results")] * inputcount
|
||||
return (error_imgs, "", "Async processing failed to produce results")
|
||||
|
||||
results_with_id.sort(key=lambda x: int(x[3])) # Sort by call_id
|
||||
|
||||
output_images = []
|
||||
output_urls = []
|
||||
output_texts = []
|
||||
|
||||
for img_tensor, image_url, response_text, call_id_res in results_with_id:
|
||||
output_images.append(img_tensor)
|
||||
output_urls.append(f"Input {call_id_res}: {image_url}")
|
||||
output_texts.append(f"Response for Input {call_id_res}: {response_text}")
|
||||
|
||||
# Handle variable image sizes by finding the maximum dimensions
|
||||
if output_images:
|
||||
# Find max dimensions across all images
|
||||
max_height = max(img.shape[1] for img in output_images)
|
||||
max_width = max(img.shape[2] for img in output_images)
|
||||
|
||||
self._log(f"Max dimensions found: {max_height}x{max_width}")
|
||||
|
||||
# Resize all images to max dimensions with padding
|
||||
resized_images = []
|
||||
for i, img_tensor in enumerate(output_images):
|
||||
current_h, current_w = img_tensor.shape[1], img_tensor.shape[2]
|
||||
|
||||
if current_h == max_height and current_w == max_width:
|
||||
# Image is already the right size
|
||||
resized_images.append(img_tensor)
|
||||
self._log(f"Image {i+1} already correct size: {current_h}x{current_w}")
|
||||
else:
|
||||
# Need to pad the image
|
||||
self._log(f"Resizing image {i+1} from {current_h}x{current_w} to {max_height}x{max_width}")
|
||||
|
||||
# Calculate padding
|
||||
pad_h = max_height - current_h
|
||||
pad_w = max_width - current_w
|
||||
|
||||
# Pad with zeros (black) - format: (pad_left, pad_right, pad_top, pad_bottom)
|
||||
pad_left = pad_w // 2
|
||||
pad_right = pad_w - pad_left
|
||||
pad_top = pad_h // 2
|
||||
pad_bottom = pad_h - pad_top
|
||||
|
||||
# PyTorch pad format: (pad_left, pad_right, pad_top, pad_bottom, pad_front, pad_back)
|
||||
# For images (B, H, W, C), we only pad H and W dimensions
|
||||
padded_img = torch.nn.functional.pad(
|
||||
img_tensor.permute(0, 3, 1, 2), # Convert to (B, C, H, W)
|
||||
(pad_left, pad_right, pad_top, pad_bottom),
|
||||
mode='constant',
|
||||
value=0
|
||||
).permute(0, 2, 3, 1) # Convert back to (B, H, W, C)
|
||||
|
||||
resized_images.append(padded_img)
|
||||
|
||||
# Now concatenate all resized images
|
||||
batched_images = torch.cat(resized_images, dim=0)
|
||||
self._log(f"Successfully batched {len(resized_images)} images with final shape: {batched_images.shape}")
|
||||
else:
|
||||
batched_images = self._create_error_image("No images generated")
|
||||
|
||||
combined_urls = " | ".join(output_urls)
|
||||
combined_responses = "\n\n".join(output_texts)
|
||||
|
||||
final_log_output = "Processing Logs:\n" + "\n".join(self.log_messages) + "\n\n" + combined_responses
|
||||
|
||||
return (batched_images, combined_urls, final_log_output)
|
||||
@@ -1,418 +0,0 @@
|
||||
# FL_Fal_Pixverse: Fal AI Image-to-Video API Node with frame decomposition
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import tempfile
|
||||
import cv2
|
||||
import concurrent.futures
|
||||
import fal_client
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class FL_Fal_Pixverse:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Image-to-Video API.
|
||||
Takes an image and converts it to a video using Fal AI's pixverse/v4/image-to-video endpoint.
|
||||
Downloads the video, extracts frames, and returns them as image tensors.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames_1", "frames_2", "frames_3", "frames_4", "frames_5", "video_urls", "status_msg")
|
||||
FUNCTION = "generate_video"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"prompt": ("STRING", {"default": "A woman walking through a beautiful landscape",
|
||||
"multiline": True, "description": "Text prompt describing the video"}),
|
||||
"aspect_ratio": (["16:9", "4:3", "1:1", "3:4", "9:16"], {"default": "16:9",
|
||||
"description": "The aspect ratio of the generated video"}),
|
||||
"resolution": (["360p", "540p", "720p", "1080p"], {"default": "720p",
|
||||
"description": "The resolution of the generated video"}),
|
||||
"duration": (["5", "8"], {"default": "5",
|
||||
"description": "Duration in seconds (8s videos cost double, 1080p limited to 5s)"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647,
|
||||
"description": "Random seed for video generation (0 = random)"}),
|
||||
"batch_size": ("INT", {"default": 1, "min": 1, "max": 5,
|
||||
"description": "Number of videos to generate with different seeds"}),
|
||||
"nth_frame": ("INT", {"default": 1, "min": 1, "max": 4,
|
||||
"description": "Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)"})
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE", {"description": "Input image to animate"}),
|
||||
"negative_prompt": ("STRING", {"default": "blurry, low quality, low resolution, pixelated, noisy, grainy, out of focus, poorly lit, poorly exposed, poorly composed, poorly framed, poorly cropped, poorly color corrected, poorly color graded",
|
||||
"multiline": True, "description": "Negative prompt"}),
|
||||
"style": (["none", "anime", "3d_animation", "clay", "comic", "cyberpunk"], {"default": "none",
|
||||
"description": "Style of the generated video"}),
|
||||
"camera_movement": (["none", "horizontal_left", "horizontal_right", "vertical_up", "vertical_down",
|
||||
"zoom_in", "zoom_out", "crane_up", "quickly_zoom_in", "quickly_zoom_out",
|
||||
"smooth_zoom_in", "camera_rotation", "robo_arm", "super_dolly_out",
|
||||
"whip_pan", "hitchcock", "left_follow", "right_follow", "pan_left",
|
||||
"pan_right", "fix_bg"], {"default": "none",
|
||||
"description": "Type of camera movement to apply"})
|
||||
}
|
||||
}
|
||||
|
||||
def generate_video(self, api_key, prompt="A woman walking through a beautiful landscape", aspect_ratio="16:9",
|
||||
resolution="720p", duration="5", seed=0, batch_size=1, nth_frame=1,
|
||||
image=None, negative_prompt="", style="none", camera_movement="none"):
|
||||
# Clear any existing FAL_KEY environment variable to prevent caching issues
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
print("[Fal Pixverse] Cleared existing FAL_KEY environment variable")
|
||||
"""
|
||||
Generate a video from an image, download it, and extract frames
|
||||
|
||||
Args:
|
||||
api_key: Fal AI API key
|
||||
prompt: Text prompt describing the video
|
||||
negative_prompt: Negative prompt
|
||||
duration: Video duration in seconds
|
||||
quality: Video quality
|
||||
seed: Random seed for video generation (0 = random)
|
||||
batch_size: Number of videos to generate with different seeds
|
||||
nth_frame: Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)
|
||||
image: (Optional) Input image tensor
|
||||
|
||||
Returns:
|
||||
Tuple of (frames_tensor_1, frames_tensor_2, frames_tensor_3, frames_tensor_4, frames_tensor_5,
|
||||
video_urls, status_message)
|
||||
Note: If batch_size < 5, the unused frame tensors will be empty (1,1,1,3) tensors
|
||||
"""
|
||||
try:
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", error_msg
|
||||
|
||||
# 1. Validate API key
|
||||
if not api_key or api_key.strip() == "":
|
||||
return error_return("Error: API Key is required")
|
||||
|
||||
# 2. Validate image input
|
||||
if image is None:
|
||||
return error_return("Error: Input image is required")
|
||||
|
||||
# Initialize return values
|
||||
frame_tensors = [torch.zeros((1, 1, 1, 3)) for _ in range(5)] # 5 empty tensors by default
|
||||
video_urls = []
|
||||
status_messages = []
|
||||
|
||||
# Limit batch size to maximum of 5
|
||||
batch_size = min(batch_size, 5)
|
||||
|
||||
# Validate duration for 1080p (limited to 5 seconds)
|
||||
if resolution == "1080p" and duration == "8":
|
||||
print(f"[Fal Pixverse] Warning: 1080p videos are limited to 5 seconds, changing duration from 8s to 5s")
|
||||
duration = "5"
|
||||
|
||||
# Convert image tensor to base64
|
||||
if image is not None:
|
||||
# Take first image if batch
|
||||
if len(image.shape) == 4:
|
||||
image_tensor = image[0]
|
||||
else:
|
||||
image_tensor = image
|
||||
|
||||
# Convert to uint8
|
||||
if image_tensor.dtype != torch.uint8:
|
||||
image_tensor = (image_tensor * 255).to(torch.uint8)
|
||||
|
||||
# Convert to numpy for PIL
|
||||
np_img = image_tensor.cpu().numpy()
|
||||
|
||||
try:
|
||||
pil_image = Image.fromarray(np_img)
|
||||
print(f"[Fal Pixverse] Successfully converted image tensor to PIL image")
|
||||
|
||||
# Set API key first - needed for CDN upload
|
||||
clean_api_key = api_key.strip()
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Upload image to fal.media CDN to avoid 10MB request body limit
|
||||
print(f"[Fal Pixverse] Uploading image to fal.media CDN...")
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
img_url = fal_client.upload(image_bytes, content_type="image/png")
|
||||
print(f"[Fal Pixverse] Uploaded image to CDN: {img_url[:80]}...")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse] Error: Failed to upload image to CDN: {str(e)}")
|
||||
return error_return(f"Error: Failed to upload image: {str(e)}")
|
||||
else:
|
||||
return error_return("Error: No image provided")
|
||||
|
||||
# Process batches in parallel
|
||||
def process_batch(batch_idx):
|
||||
try:
|
||||
# Calculate seed for this batch
|
||||
batch_seed = np.random.randint(1, 2147483647) if seed == 0 else seed + batch_idx
|
||||
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}/{batch_size}: Generating video with seed {batch_seed}...")
|
||||
|
||||
# Prepare the API request
|
||||
# Ensure API key is properly formatted (trim any whitespace)
|
||||
clean_api_key = api_key.strip()
|
||||
|
||||
# Prepare the arguments for fal_client
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"image_url": img_url,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"duration": duration,
|
||||
"seed": batch_seed
|
||||
}
|
||||
|
||||
# Add optional parameters if provided and valid
|
||||
if negative_prompt and negative_prompt.strip():
|
||||
arguments["negative_prompt"] = negative_prompt.strip()
|
||||
|
||||
if style and style != "none" and style != "":
|
||||
arguments["style"] = style
|
||||
|
||||
# Camera movement might have resolution restrictions - only add if not none and resolution is compatible
|
||||
if camera_movement and camera_movement != "none" and camera_movement != "":
|
||||
# Some camera movements might have resolution limits - let's be more conservative
|
||||
if resolution in ["360p", "540p"]:
|
||||
arguments["camera_movement"] = camera_movement
|
||||
print(f"[Fal Pixverse] Adding camera_movement: {camera_movement} (resolution: {resolution})")
|
||||
else:
|
||||
print(f"[Fal Pixverse] Skipping camera_movement for {resolution} - may have restrictions")
|
||||
|
||||
# Keep duration as string per API documentation
|
||||
arguments["duration"] = str(duration)
|
||||
|
||||
# Remove any None values from arguments
|
||||
arguments = {k: v for k, v in arguments.items() if v is not None}
|
||||
|
||||
# Print arguments without exposing potentially large base64 data
|
||||
safe_arguments = {k: v if not (isinstance(v, str) and v.startswith('data:')) else f"<data_uri_{len(v)}_chars>" for k, v in arguments.items()}
|
||||
print(f"[Fal Pixverse] API arguments: {safe_arguments}")
|
||||
|
||||
# Set the API key as an environment variable for fal_client (using cleaned key)
|
||||
# Print the first few characters of the key for debugging (don't print the whole key for security)
|
||||
key_preview = clean_api_key[:8] + "..." if len(clean_api_key) > 8 else "invalid_key"
|
||||
print(f"[Fal Pixverse] Using API key starting with: {key_preview}")
|
||||
|
||||
# Clear and set the environment variable
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
print(f"[Fal Pixverse] Calling Fal AI API with fal_client...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
print(f"[Fal Pixverse] Log: {log['message']}")
|
||||
|
||||
try:
|
||||
# Use the new v5 endpoint
|
||||
endpoint = "fal-ai/pixverse/v5/image-to-video"
|
||||
print(f"[Fal Pixverse] Using v5 endpoint: {endpoint}")
|
||||
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
print(f"[Fal Pixverse] Making API call with fal_client.subscribe...")
|
||||
result = fal_client.subscribe(
|
||||
endpoint,
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
|
||||
print(f"[Fal Pixverse] API call completed successfully")
|
||||
except Exception as e:
|
||||
error_msg = f"API Error: {str(e)}"
|
||||
print(f"[Fal Pixverse] {error_msg}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": error_msg
|
||||
}
|
||||
|
||||
# Extract video URL from the result
|
||||
if "video" in result and "url" in result["video"]:
|
||||
video_url = result["video"]["url"]
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Video ready! URL: {video_url}")
|
||||
|
||||
# Download and process the video
|
||||
try:
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Downloading video...")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
# Download video to temp file
|
||||
dl_response = requests.get(video_url, stream=True)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Get file size for progress bar
|
||||
file_size = int(dl_response.headers.get('content-length', 0))
|
||||
progress_bar = tqdm(total=file_size, unit='B', unit_scale=True, desc=f"Downloading Batch {batch_idx+1}")
|
||||
|
||||
for chunk in dl_response.iter_content(chunk_size=8192):
|
||||
temp_video.write(chunk)
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Extract frames using OpenCV
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Extracting frames from video...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
os.unlink(temp_video_path) # Clean up temp file
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "Could not open video file"
|
||||
}
|
||||
|
||||
# Get video properties
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Video has {total_frames} frames at {fps} FPS")
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
# Use nth_frame directly as the stride
|
||||
stride = nth_frame
|
||||
|
||||
# Calculate approximately how many frames we'll extract
|
||||
frames_to_extract = total_frames // stride + (1 if total_frames % stride > 0 else 0)
|
||||
|
||||
progress_bar = tqdm(total=frames_to_extract, desc=f"Extracting frames (Batch {batch_idx+1})")
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % stride == 0 and len(frames) < frames_to_extract:
|
||||
# Convert BGR to RGB (OpenCV uses BGR by default)
|
||||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Normalize to 0-1 range for ComfyUI
|
||||
normalized_frame = rgb_frame.astype(np.float32) / 255.0
|
||||
|
||||
frames.append(normalized_frame)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Break if we've extracted enough frames
|
||||
if len(frames) >= frames_to_extract:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
|
||||
progress_bar.close()
|
||||
cap.release()
|
||||
|
||||
# Clean up temp file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[Fal Pixverse] Batch {batch_idx+1}: Extracted {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": True,
|
||||
"frames_tensor": frames_tensor,
|
||||
"video_url": video_url
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No frames could be extracted"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Processing Error: {str(e)}"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No video URL in API response"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Batch processing error: {str(e)}"
|
||||
}
|
||||
|
||||
# Process batches in parallel
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=batch_size) as executor:
|
||||
future_to_batch = {
|
||||
executor.submit(process_batch, idx): idx
|
||||
for idx in range(batch_size)
|
||||
}
|
||||
|
||||
for future in concurrent.futures.as_completed(future_to_batch):
|
||||
batch_idx = future_to_batch[future]
|
||||
try:
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Thread Error: {str(e)}"
|
||||
})
|
||||
|
||||
# Collect results
|
||||
for result in results:
|
||||
batch_idx = result["batch_idx"]
|
||||
if result["success"]:
|
||||
frame_tensors[batch_idx] = result["frames_tensor"]
|
||||
video_urls.append(f"Batch {batch_idx+1}: {result['video_url']}")
|
||||
status_messages.append(f"Success (Batch {batch_idx+1})")
|
||||
else:
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): {result['error']}")
|
||||
|
||||
# Combine status messages
|
||||
combined_status = " | ".join(status_messages) if status_messages else "No videos processed"
|
||||
|
||||
# Combine video URLs
|
||||
combined_urls = " | ".join(video_urls) if video_urls else "No videos generated"
|
||||
|
||||
# Return the results
|
||||
return tuple(frame_tensors + [combined_urls, combined_status])
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse] Error: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", f"Error: {str(e)}"
|
||||
@@ -1,395 +0,0 @@
|
||||
# FL_Fal_Pixverse_LipSync: Fal AI Pixverse LipSync API Node
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import tempfile
|
||||
import cv2
|
||||
import base64
|
||||
import fal_client
|
||||
import torchaudio
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class FL_Fal_Pixverse_LipSync:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Pixverse LipSync API.
|
||||
Takes a video and audio/text and generates realistic lipsync animations.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames", "audio", "video_url", "status_msg")
|
||||
FUNCTION = "generate_lipsync"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
voice_options = [
|
||||
"Auto", "Emily", "James", "Isabella", "Liam", "Sophia", "Alexander",
|
||||
"Ava", "Benjamin", "Charlotte", "Daniel", "Emma", "Gabriel", "Grace",
|
||||
"Henry", "Luna", "Jackson", "Mia", "Lucas", "Olivia", "Matthew",
|
||||
"Zoe", "Michael", "Aria", "Owen", "Chloe", "Samuel", "Lily"
|
||||
]
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"frames": ("IMAGE", {"description": "Input video frames as image sequence"}),
|
||||
"mode": (["audio_input", "text_to_speech"], {"default": "audio_input",
|
||||
"description": "Input mode: use audio file or text-to-speech"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 999999,
|
||||
"description": "Random seed (0 = random, max 6 digits)"}),
|
||||
"nth_frame": ("INT", {"default": 1, "min": 1, "max": 4,
|
||||
"description": "Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)"})
|
||||
},
|
||||
"optional": {
|
||||
"audio": ("AUDIO", {"description": "Input audio tensor (for audio_input mode)"}),
|
||||
"text": ("STRING", {"default": "Hello, this is a test message.",
|
||||
"multiline": True, "description": "Text for speech synthesis (for text_to_speech mode)"}),
|
||||
"voice_id": (voice_options, {"default": "Auto",
|
||||
"description": "Voice selection for text-to-speech"})
|
||||
}
|
||||
}
|
||||
|
||||
def sanitize_error_message(self, msg):
|
||||
"""Remove potentially large base64 data from error messages"""
|
||||
if not isinstance(msg, str):
|
||||
msg = str(msg)
|
||||
# Replace data URIs with placeholders to avoid wall of text
|
||||
import re
|
||||
msg = re.sub(r'data:[^;]+;base64,[A-Za-z0-9+/=]{100,}', '<data_uri_removed>', msg)
|
||||
return msg
|
||||
|
||||
def generate_lipsync(self, api_key, frames, mode="audio_input", seed=0, nth_frame=1,
|
||||
audio=None, text="Hello, this is a test message.", voice_id="Auto"):
|
||||
"""
|
||||
Generate lipsync video using Fal AI Pixverse LipSync API
|
||||
|
||||
Args:
|
||||
api_key: Fal AI API key
|
||||
frames: Input video frames as image sequence
|
||||
mode: Input mode - "audio_input" or "text_to_speech"
|
||||
seed: Random seed for generation (0 = random, max 6 digits)
|
||||
nth_frame: Extract every Nth frame
|
||||
audio: Input audio tensor (for audio_input mode)
|
||||
text: Text for speech synthesis (for text_to_speech mode)
|
||||
voice_id: Voice selection for text-to-speech
|
||||
|
||||
Returns:
|
||||
Tuple of (frames_tensor, audio, video_url, status_message)
|
||||
"""
|
||||
try:
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
empty_audio = {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
clean_msg = self.sanitize_error_message(error_msg)
|
||||
return empty_tensor, empty_audio, "", clean_msg
|
||||
|
||||
# Clear any existing FAL_KEY environment variable to prevent caching issues
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
print("[Fal Pixverse LipSync] Cleared existing FAL_KEY environment variable")
|
||||
|
||||
# 1. Validate API key
|
||||
if not api_key or api_key.strip() == "":
|
||||
return error_return("Error: API Key is required")
|
||||
|
||||
# 2. Validate frames input
|
||||
if frames is None:
|
||||
return error_return("Error: Frames input is required")
|
||||
|
||||
# 3. Process seed (ensure it's within 6-digit limit)
|
||||
if seed < 0 or seed > 999999:
|
||||
return error_return("Error: Seed must be between 0 and 999999 (6 digits max)")
|
||||
|
||||
# 4. Check audio input and validate mode - defer upload until after API key setup
|
||||
audio_tensor_to_process = None
|
||||
if mode == "audio_input":
|
||||
if audio is not None:
|
||||
audio_tensor_to_process = audio
|
||||
print(f"[Fal Pixverse LipSync] Audio tensor provided, will process after API setup")
|
||||
else:
|
||||
return error_return("Error: Audio tensor is required for audio_input mode")
|
||||
elif mode == "text_to_speech":
|
||||
if not text or text.strip() == "":
|
||||
return error_return("Error: Text is required for text_to_speech mode")
|
||||
else:
|
||||
return error_return("Error: Invalid mode. Must be 'audio_input' or 'text_to_speech'")
|
||||
|
||||
# 5. Set up frames for processing (convert to video after API key setup)
|
||||
frames_to_process = frames
|
||||
print(f"[Fal Pixverse LipSync] Frames provided, will convert to video after API setup")
|
||||
print(f"[Fal Pixverse LipSync] Using seed: {seed}")
|
||||
|
||||
print(f"[Fal Pixverse LipSync] Starting lipsync generation...")
|
||||
print(f"[Fal Pixverse LipSync] Mode: {mode}")
|
||||
|
||||
# Prepare the API request
|
||||
clean_api_key = api_key.strip()
|
||||
|
||||
# Prepare the arguments for fal_client (video_url will be set after frames conversion)
|
||||
arguments = {
|
||||
"video_url": "" # Will be set after frames to video conversion
|
||||
}
|
||||
|
||||
# Add seed if specified (0 means random)
|
||||
if seed > 0:
|
||||
arguments["seed"] = seed
|
||||
|
||||
# Add mode-specific parameters (URLs will be set after upload)
|
||||
if mode == "audio_input":
|
||||
arguments["audio_url"] = "" # Will be set after audio upload
|
||||
elif mode == "text_to_speech":
|
||||
arguments["text"] = text.strip()
|
||||
arguments["voice_id"] = voice_id
|
||||
print(f"[Fal Pixverse LipSync] Text: {text[:50]}{'...' if len(text) > 50 else ''}")
|
||||
print(f"[Fal Pixverse LipSync] Voice ID: {voice_id}")
|
||||
|
||||
# Remove any None values from arguments
|
||||
arguments = {k: v for k, v in arguments.items() if v is not None and v != ""}
|
||||
|
||||
# Print arguments without exposing potentially large base64 data
|
||||
safe_arguments = {k: v if not (isinstance(v, str) and v.startswith('data:')) else f"<data_uri_{len(v)}_chars>" for k, v in arguments.items()}
|
||||
print(f"[Fal Pixverse LipSync] API arguments: {safe_arguments}")
|
||||
|
||||
# Set the API key as an environment variable for fal_client
|
||||
key_preview = clean_api_key[:8] + "..." if len(clean_api_key) > 8 else "invalid_key"
|
||||
print(f"[Fal Pixverse LipSync] Using API key starting with: {key_preview}")
|
||||
|
||||
# Clear and set the environment variable
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
print(f"[Fal Pixverse LipSync] Calling Fal AI API with fal_client...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
print(f"[Fal Pixverse LipSync] Log: {log['message']}")
|
||||
|
||||
try:
|
||||
# Use the lipsync endpoint
|
||||
endpoint = "fal-ai/pixverse/lipsync"
|
||||
print(f"[Fal Pixverse LipSync] Using endpoint: {endpoint}")
|
||||
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Process audio tensor upload if needed (after fal_client is properly loaded)
|
||||
if audio_tensor_to_process is not None:
|
||||
try:
|
||||
# Convert audio tensor to temporary file and upload to Fal
|
||||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as temp_audio:
|
||||
temp_audio_path = temp_audio.name
|
||||
|
||||
# Save audio tensor to temporary file
|
||||
waveform = audio_tensor_to_process['waveform']
|
||||
sample_rate = audio_tensor_to_process['sample_rate']
|
||||
|
||||
# Ensure waveform is in correct format for torchaudio.save
|
||||
if len(waveform.shape) == 3: # [batch, channels, samples]
|
||||
waveform = waveform.squeeze(0) # Remove batch dimension
|
||||
|
||||
torchaudio.save(temp_audio_path, waveform, sample_rate)
|
||||
print(f"[Fal Pixverse LipSync] Saved audio tensor to temporary file: {temp_audio_path}")
|
||||
|
||||
# Upload to Fal
|
||||
final_audio_url = fal_client.upload_file(temp_audio_path)
|
||||
print(f"[Fal Pixverse LipSync] Uploaded audio to Fal: {final_audio_url}")
|
||||
|
||||
# Update arguments with the uploaded audio URL
|
||||
arguments["audio_url"] = final_audio_url
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_audio_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse LipSync] Error processing audio tensor: {str(e)}")
|
||||
return error_return(f"Error: Failed to process audio: {str(e)}")
|
||||
|
||||
# Process frames to video conversion and upload (after fal_client is properly loaded)
|
||||
try:
|
||||
# Convert frames to temporary video file and upload to Fal
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
print(f"[Fal Pixverse LipSync] Converting frames to video...")
|
||||
|
||||
# Convert frames tensor to video using OpenCV
|
||||
if len(frames_to_process.shape) == 4: # [batch, height, width, channels]
|
||||
frames_np = frames_to_process.cpu().numpy()
|
||||
else:
|
||||
frames_np = frames_to_process.unsqueeze(0).cpu().numpy()
|
||||
|
||||
# Convert to uint8 if needed
|
||||
if frames_np.dtype != np.uint8:
|
||||
frames_np = (frames_np * 255).astype(np.uint8)
|
||||
|
||||
# Get video properties
|
||||
batch_size, height, width, channels = frames_np.shape
|
||||
fps = 24 # Default FPS for video
|
||||
|
||||
# Create video writer
|
||||
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
|
||||
out = cv2.VideoWriter(temp_video_path, fourcc, fps, (width, height))
|
||||
|
||||
# Write frames to video
|
||||
for i in range(batch_size):
|
||||
frame = frames_np[i]
|
||||
# Convert RGB to BGR for OpenCV
|
||||
if channels == 3:
|
||||
frame_bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
|
||||
else:
|
||||
frame_bgr = frame
|
||||
out.write(frame_bgr)
|
||||
|
||||
out.release()
|
||||
print(f"[Fal Pixverse LipSync] Saved frames to temporary video: {temp_video_path}")
|
||||
|
||||
# Upload video to Fal
|
||||
video_url = fal_client.upload_file(temp_video_path)
|
||||
print(f"[Fal Pixverse LipSync] Uploaded video to Fal: {video_url}")
|
||||
|
||||
# Update arguments with the uploaded video URL
|
||||
arguments["video_url"] = video_url
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse LipSync] Error processing frames to video: {str(e)}")
|
||||
return error_return(f"Error: Failed to convert frames to video: {str(e)}")
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
print(f"[Fal Pixverse LipSync] Making API call with fal_client.subscribe...")
|
||||
result = fal_client.subscribe(
|
||||
endpoint,
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
|
||||
print(f"[Fal Pixverse LipSync] API call completed successfully")
|
||||
except Exception as e:
|
||||
error_msg = f"API Error: {str(e)}"
|
||||
print(f"[Fal Pixverse LipSync] {error_msg}")
|
||||
return error_return(error_msg)
|
||||
|
||||
# Extract video URL from the result
|
||||
if "video" in result and "url" in result["video"]:
|
||||
output_video_url = result["video"]["url"]
|
||||
print(f"[Fal Pixverse LipSync] Video ready! URL: {output_video_url}")
|
||||
|
||||
# Download and process the video
|
||||
try:
|
||||
print(f"[Fal Pixverse LipSync] Downloading video...")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
# Download video to temp file
|
||||
dl_response = requests.get(output_video_url, stream=True)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Get file size for progress bar
|
||||
file_size = int(dl_response.headers.get('content-length', 0))
|
||||
progress_bar = tqdm(total=file_size, unit='B', unit_scale=True, desc="Downloading LipSync Video")
|
||||
|
||||
for chunk in dl_response.iter_content(chunk_size=8192):
|
||||
temp_video.write(chunk)
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Extract frames using OpenCV
|
||||
print(f"[Fal Pixverse LipSync] Extracting frames from video...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
os.unlink(temp_video_path) # Clean up temp file
|
||||
return error_return("Could not open video file")
|
||||
|
||||
# Get video properties
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
print(f"[Fal Pixverse LipSync] Video has {total_frames} frames at {fps} FPS")
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
# Use nth_frame directly as the stride
|
||||
stride = nth_frame
|
||||
|
||||
# Calculate approximately how many frames we'll extract
|
||||
frames_to_extract = total_frames // stride + (1 if total_frames % stride > 0 else 0)
|
||||
|
||||
progress_bar = tqdm(total=frames_to_extract, desc="Extracting frames")
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % stride == 0 and len(frames) < frames_to_extract:
|
||||
# Convert BGR to RGB (OpenCV uses BGR by default)
|
||||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Normalize to 0-1 range for ComfyUI
|
||||
normalized_frame = rgb_frame.astype(np.float32) / 255.0
|
||||
|
||||
frames.append(normalized_frame)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Break if we've extracted enough frames
|
||||
if len(frames) >= frames_to_extract:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
|
||||
progress_bar.close()
|
||||
cap.release()
|
||||
|
||||
# Clean up temp file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[Fal Pixverse LipSync] Extracted {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
|
||||
# Prepare audio output - use the original input audio if provided, otherwise empty audio
|
||||
output_audio = audio if audio is not None else {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
|
||||
return frames_tensor, output_audio, output_video_url, "Success: LipSync video generated and processed"
|
||||
else:
|
||||
return error_return("No frames could be extracted")
|
||||
|
||||
except Exception as e:
|
||||
return error_return(f"Processing Error: {str(e)}")
|
||||
else:
|
||||
return error_return("No video URL in API response")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse LipSync] Error: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
empty_audio = {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
# Sanitize error message to remove potential base64 data
|
||||
clean_error = self.sanitize_error_message(f"Error: {str(e)}")
|
||||
return empty_tensor, empty_audio, "", clean_error
|
||||
@@ -1,409 +0,0 @@
|
||||
# FL_Fal_Pixverse_Transition: Fal AI Pixverse v5 Transition API Node with frame decomposition
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import tempfile
|
||||
import cv2
|
||||
import concurrent.futures
|
||||
import fal_client
|
||||
import asyncio
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
class FL_Fal_Pixverse_Transition:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI Pixverse v5 Transition API.
|
||||
Takes two images and creates a transition video between them using Fal AI's transition endpoint.
|
||||
Downloads the video, extracts frames, and returns them as image tensors.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "IMAGE", "IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames_1", "frames_2", "frames_3", "frames_4", "frames_5", "video_urls", "status_msg")
|
||||
FUNCTION = "generate_transition"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"prompt": ("STRING", {"default": "Scene slowly transition into cat swimming under water",
|
||||
"multiline": True, "description": "The prompt for the transition"}),
|
||||
"aspect_ratio": (["16:9", "4:3", "1:1", "3:4", "9:16"], {"default": "16:9",
|
||||
"description": "The aspect ratio of the generated video"}),
|
||||
"resolution": (["360p", "540p", "720p", "1080p"], {"default": "720p",
|
||||
"description": "The resolution of the generated video"}),
|
||||
"duration": (["5", "8"], {"default": "5",
|
||||
"description": "Duration in seconds (8s videos cost double, 1080p limited to 5s)"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647,
|
||||
"description": "Random seed for video generation (0 = random)"}),
|
||||
"batch_size": ("INT", {"default": 1, "min": 1, "max": 5,
|
||||
"description": "Number of videos to generate with different seeds"}),
|
||||
"nth_frame": ("INT", {"default": 1, "min": 1, "max": 4,
|
||||
"description": "Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)"})
|
||||
},
|
||||
"optional": {
|
||||
"first_image": ("IMAGE", {"description": "First input image (start of transition)"}),
|
||||
"last_image": ("IMAGE", {"description": "Last input image (end of transition)"}),
|
||||
"negative_prompt": ("STRING", {"default": "blurry, low quality, low resolution, pixelated, noisy, grainy, out of focus, poorly lit, poorly exposed, poorly composed, poorly framed, poorly cropped, poorly color corrected, poorly color graded",
|
||||
"multiline": True, "description": "Negative prompt"}),
|
||||
"style": (["none", "anime", "3d_animation", "clay", "comic", "cyberpunk"], {"default": "none",
|
||||
"description": "Style of the generated video"})
|
||||
}
|
||||
}
|
||||
|
||||
def generate_transition(self, api_key, prompt="Scene slowly transition into cat swimming under water", aspect_ratio="16:9",
|
||||
resolution="720p", duration="5", seed=0, batch_size=1, nth_frame=1,
|
||||
first_image=None, last_image=None, negative_prompt="", style="none"):
|
||||
# Clear any existing FAL_KEY environment variable to prevent caching issues
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
print("[Fal Pixverse Transition] Cleared existing FAL_KEY environment variable")
|
||||
"""
|
||||
Generate a transition video between two images, download it, and extract frames
|
||||
|
||||
Args:
|
||||
api_key: Fal AI API key
|
||||
prompt: Text prompt describing the transition
|
||||
aspect_ratio: Aspect ratio of the video
|
||||
resolution: Video resolution
|
||||
duration: Video duration in seconds
|
||||
seed: Random seed for video generation (0 = random)
|
||||
batch_size: Number of videos to generate with different seeds
|
||||
nth_frame: Extract every Nth frame (1=all frames, 2=every 2nd frame, etc.)
|
||||
first_image: First input image tensor (start of transition)
|
||||
last_image: Last input image tensor (end of transition)
|
||||
negative_prompt: Negative prompt
|
||||
style: Video style
|
||||
|
||||
Returns:
|
||||
Tuple of (frames_tensor_1, frames_tensor_2, frames_tensor_3, frames_tensor_4, frames_tensor_5,
|
||||
video_urls, status_message)
|
||||
Note: If batch_size < 5, the unused frame tensors will be empty (1,1,1,3) tensors
|
||||
"""
|
||||
try:
|
||||
# Helper function for error returns
|
||||
def error_return(error_msg):
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", error_msg
|
||||
|
||||
# 1. Validate API key
|
||||
if not api_key or api_key.strip() == "":
|
||||
return error_return("Error: API Key is required")
|
||||
|
||||
# 2. Validate image inputs
|
||||
if first_image is None or last_image is None:
|
||||
return error_return("Error: Both first_image and last_image are required for transition")
|
||||
|
||||
# Initialize return values
|
||||
frame_tensors = [torch.zeros((1, 1, 1, 3)) for _ in range(5)] # 5 empty tensors by default
|
||||
video_urls = []
|
||||
status_messages = []
|
||||
|
||||
# Limit batch size to maximum of 5
|
||||
batch_size = min(batch_size, 5)
|
||||
|
||||
# Validate duration for 1080p (limited to 5 seconds)
|
||||
if resolution == "1080p" and duration == "8":
|
||||
print(f"[Fal Pixverse Transition] Warning: 1080p videos are limited to 5 seconds, changing duration from 8s to 5s")
|
||||
duration = "5"
|
||||
|
||||
# Set API key FIRST - needed for CDN upload
|
||||
clean_api_key = api_key.strip()
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Upload images to fal.media CDN
|
||||
def upload_image_to_fal(image_tensor, image_name):
|
||||
# Take first image if batch
|
||||
if len(image_tensor.shape) == 4:
|
||||
image_tensor = image_tensor[0]
|
||||
|
||||
# Convert to uint8
|
||||
if image_tensor.dtype != torch.uint8:
|
||||
image_tensor = (image_tensor * 255).to(torch.uint8)
|
||||
|
||||
# Convert to numpy for PIL
|
||||
np_img = image_tensor.cpu().numpy()
|
||||
|
||||
try:
|
||||
pil_image = Image.fromarray(np_img)
|
||||
print(f"[Fal Pixverse Transition] Successfully converted {image_name} to PIL image")
|
||||
|
||||
# Upload to fal.media CDN
|
||||
print(f"[Fal Pixverse Transition] Uploading {image_name} to fal.media CDN...")
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
url = fal_client.upload(image_bytes, content_type="image/png")
|
||||
print(f"[Fal Pixverse Transition] Uploaded {image_name} to CDN: {url[:80]}...")
|
||||
return url
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse Transition] Error: Failed to upload {image_name} to CDN: {str(e)}")
|
||||
raise Exception(f"Failed to upload {image_name}: {str(e)}")
|
||||
|
||||
# Upload both images to CDN
|
||||
first_image_url = upload_image_to_fal(first_image, "first_image")
|
||||
last_image_url = upload_image_to_fal(last_image, "last_image")
|
||||
|
||||
# Process batches in parallel
|
||||
def process_batch(batch_idx):
|
||||
try:
|
||||
# Calculate seed for this batch
|
||||
batch_seed = np.random.randint(1, 2147483647) if seed == 0 else seed + batch_idx
|
||||
|
||||
print(f"[Fal Pixverse Transition] Batch {batch_idx+1}/{batch_size}: Generating transition video with seed {batch_seed}...")
|
||||
|
||||
# Prepare the API request
|
||||
# Ensure API key is properly formatted (trim any whitespace)
|
||||
clean_api_key = api_key.strip()
|
||||
|
||||
# Prepare the arguments for fal_client
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"first_image_url": first_image_url,
|
||||
"last_image_url": last_image_url,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"resolution": resolution,
|
||||
"duration": duration,
|
||||
"seed": batch_seed
|
||||
}
|
||||
|
||||
# Add optional parameters if provided and valid
|
||||
if negative_prompt and negative_prompt.strip():
|
||||
arguments["negative_prompt"] = negative_prompt.strip()
|
||||
|
||||
if style and style != "none" and style != "":
|
||||
arguments["style"] = style
|
||||
|
||||
# Keep duration as string per API documentation
|
||||
arguments["duration"] = str(duration)
|
||||
|
||||
# Remove any None values from arguments
|
||||
arguments = {k: v for k, v in arguments.items() if v is not None}
|
||||
|
||||
# Set the API key as an environment variable for fal_client (using cleaned key)
|
||||
# Print the first few characters of the key for debugging (don't print the whole key for security)
|
||||
key_preview = clean_api_key[:8] + "..." if len(clean_api_key) > 8 else "invalid_key"
|
||||
print(f"[Fal Pixverse Transition] Using API key starting with: {key_preview}")
|
||||
|
||||
# Clear and set the environment variable
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Print arguments without exposing potentially large base64 data
|
||||
safe_arguments = {k: v if not (isinstance(v, str) and v.startswith('data:')) else f"<data_uri_{len(v)}_chars>" for k, v in arguments.items()}
|
||||
print(f"[Fal Pixverse Transition] API arguments: {safe_arguments}")
|
||||
print(f"[Fal Pixverse Transition] Calling Fal AI API with fal_client...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
print(f"[Fal Pixverse Transition] Log: {log['message']}")
|
||||
|
||||
try:
|
||||
# Use the v5 transition endpoint
|
||||
endpoint = "fal-ai/pixverse/v5/transition"
|
||||
print(f"[Fal Pixverse Transition] Using transition endpoint: {endpoint}")
|
||||
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
print(f"[Fal Pixverse Transition] Making API call with fal_client.subscribe...")
|
||||
result = fal_client.subscribe(
|
||||
endpoint,
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
|
||||
print(f"[Fal Pixverse Transition] API call completed successfully")
|
||||
except Exception as e:
|
||||
error_msg = f"API Error: {str(e)}"
|
||||
print(f"[Fal Pixverse Transition] {error_msg}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": error_msg
|
||||
}
|
||||
|
||||
# Extract video URL from the result
|
||||
if "video" in result and "url" in result["video"]:
|
||||
video_url = result["video"]["url"]
|
||||
print(f"[Fal Pixverse Transition] Batch {batch_idx+1}: Video ready! URL: {video_url}")
|
||||
|
||||
# Download and process the video
|
||||
try:
|
||||
print(f"[Fal Pixverse Transition] Batch {batch_idx+1}: Downloading video...")
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_video:
|
||||
temp_video_path = temp_video.name
|
||||
|
||||
# Download video to temp file
|
||||
dl_response = requests.get(video_url, stream=True)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Get file size for progress bar
|
||||
file_size = int(dl_response.headers.get('content-length', 0))
|
||||
progress_bar = tqdm(total=file_size, unit='B', unit_scale=True, desc=f"Downloading Batch {batch_idx+1}")
|
||||
|
||||
for chunk in dl_response.iter_content(chunk_size=8192):
|
||||
temp_video.write(chunk)
|
||||
progress_bar.update(len(chunk))
|
||||
|
||||
progress_bar.close()
|
||||
|
||||
# Extract frames using OpenCV
|
||||
print(f"[Fal Pixverse Transition] Batch {batch_idx+1}: Extracting frames from video...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
os.unlink(temp_video_path) # Clean up temp file
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "Could not open video file"
|
||||
}
|
||||
|
||||
# Get video properties
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
print(f"[Fal Pixverse Transition] Batch {batch_idx+1}: Video has {total_frames} frames at {fps} FPS")
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
# Use nth_frame directly as the stride
|
||||
stride = nth_frame
|
||||
|
||||
# Calculate approximately how many frames we'll extract
|
||||
frames_to_extract = total_frames // stride + (1 if total_frames % stride > 0 else 0)
|
||||
|
||||
progress_bar = tqdm(total=frames_to_extract, desc=f"Extracting frames (Batch {batch_idx+1})")
|
||||
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % stride == 0 and len(frames) < frames_to_extract:
|
||||
# Convert BGR to RGB (OpenCV uses BGR by default)
|
||||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Normalize to 0-1 range for ComfyUI
|
||||
normalized_frame = rgb_frame.astype(np.float32) / 255.0
|
||||
|
||||
frames.append(normalized_frame)
|
||||
progress_bar.update(1)
|
||||
|
||||
# Break if we've extracted enough frames
|
||||
if len(frames) >= frames_to_extract:
|
||||
break
|
||||
|
||||
frame_count += 1
|
||||
|
||||
progress_bar.close()
|
||||
cap.release()
|
||||
|
||||
# Clean up temp file
|
||||
os.unlink(temp_video_path)
|
||||
|
||||
# Convert frames to tensor
|
||||
if frames:
|
||||
frames_tensor = torch.from_numpy(np.stack(frames))
|
||||
print(f"[Fal Pixverse Transition] Batch {batch_idx+1}: Extracted {len(frames)} frames as tensor with shape {frames_tensor.shape}")
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": True,
|
||||
"frames_tensor": frames_tensor,
|
||||
"video_url": video_url
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No frames could be extracted"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Processing Error: {str(e)}"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": "No video URL in API response"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Batch processing error: {str(e)}"
|
||||
}
|
||||
|
||||
# Process batches in parallel
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=batch_size) as executor:
|
||||
future_to_batch = {
|
||||
executor.submit(process_batch, idx): idx
|
||||
for idx in range(batch_size)
|
||||
}
|
||||
|
||||
for future in concurrent.futures.as_completed(future_to_batch):
|
||||
batch_idx = future_to_batch[future]
|
||||
try:
|
||||
result = future.result()
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"batch_idx": batch_idx,
|
||||
"success": False,
|
||||
"error": f"Thread Error: {str(e)}"
|
||||
})
|
||||
|
||||
# Collect results
|
||||
for result in results:
|
||||
batch_idx = result["batch_idx"]
|
||||
if result["success"]:
|
||||
frame_tensors[batch_idx] = result["frames_tensor"]
|
||||
video_urls.append(f"Batch {batch_idx+1}: {result['video_url']}")
|
||||
status_messages.append(f"Success (Batch {batch_idx+1})")
|
||||
else:
|
||||
video_urls.append(f"Batch {batch_idx+1}: Failed")
|
||||
status_messages.append(f"Error (Batch {batch_idx+1}): {result['error']}")
|
||||
|
||||
# Combine status messages
|
||||
combined_status = " | ".join(status_messages) if status_messages else "No videos processed"
|
||||
|
||||
# Combine video URLs
|
||||
combined_urls = " | ".join(video_urls) if video_urls else "No videos generated"
|
||||
|
||||
# Return the results
|
||||
return tuple(frame_tensors + [combined_urls, combined_status])
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal Pixverse Transition] Error: {str(e)}")
|
||||
# Try to return proper empty tensors
|
||||
empty_tensor = torch.zeros((1, 1, 1, 3))
|
||||
return empty_tensor, empty_tensor, empty_tensor, empty_tensor, empty_tensor, "", f"Error: {str(e)}"
|
||||
@@ -1,221 +0,0 @@
|
||||
import io
|
||||
import numpy as np
|
||||
import os
|
||||
import requests
|
||||
from PIL import Image
|
||||
import torch
|
||||
from tqdm import tqdm
|
||||
import fal_client
|
||||
|
||||
|
||||
class FL_Fal_SeedVR_Upscale:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {
|
||||
"multiline": False,
|
||||
"default": "",
|
||||
"placeholder": "Enter your Fal AI API key"
|
||||
}),
|
||||
"image": ("IMAGE",),
|
||||
"upscale_factor": ("FLOAT", {
|
||||
"default": 2.0,
|
||||
"min": 1.0,
|
||||
"max": 10.0,
|
||||
"step": 0.5,
|
||||
"description": "Upscaling factor (multiplies dimensions)"
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": -1,
|
||||
"min": -1,
|
||||
"max": 999999999,
|
||||
"description": "Random seed (-1 for random)"
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "INT", "INT")
|
||||
RETURN_NAMES = ("upscaled_image", "image_url", "width", "height")
|
||||
FUNCTION = "upscale_image"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def upload_tensor_to_fal(self, tensor_image):
|
||||
"""Upload tensor image to fal.media CDN and return the URL."""
|
||||
# ComfyUI tensors are in [B, H, W, C] or [H, W, C] format
|
||||
# Remove batch dimension if present
|
||||
if tensor_image.dim() == 4:
|
||||
tensor_image = tensor_image.squeeze(0)
|
||||
|
||||
# Ensure values are in [0, 1] range and convert to [0, 255]
|
||||
tensor_image = (tensor_image * 255.0).clamp(0, 255)
|
||||
|
||||
# Convert to uint8 and then to PIL Image
|
||||
np_image = tensor_image.cpu().numpy().astype(np.uint8)
|
||||
pil_image = Image.fromarray(np_image)
|
||||
|
||||
# Upload to fal.media CDN
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
url = fal_client.upload(image_bytes, content_type="image/png")
|
||||
print(f"[Fal SeedVR Upscale] Uploaded image to CDN: {url[:80]}...")
|
||||
return url
|
||||
|
||||
def download_image(self, image_url):
|
||||
"""Download image from URL and convert to tensor"""
|
||||
try:
|
||||
print(f"[Fal SeedVR Upscale] Downloading upscaled image...")
|
||||
|
||||
# Download image with progress bar
|
||||
response = requests.get(image_url, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
|
||||
image_data = io.BytesIO()
|
||||
with tqdm(total=total_size, unit='iB', unit_scale=True, desc="Downloading image") as pbar:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
image_data.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
|
||||
# Load image
|
||||
image_data.seek(0)
|
||||
pil_image = Image.open(image_data).convert('RGB')
|
||||
|
||||
# Convert to tensor
|
||||
tensor_image = torch.from_numpy(np.array(pil_image)).float() / 255.0
|
||||
tensor_image = tensor_image.unsqueeze(0) # Add batch dimension
|
||||
|
||||
print(f"[Fal SeedVR Upscale] Image downloaded successfully: {pil_image.width}x{pil_image.height}")
|
||||
|
||||
return tensor_image, pil_image.width, pil_image.height
|
||||
|
||||
except Exception as e:
|
||||
print(f"[Fal SeedVR Upscale] Error downloading image: {str(e)}")
|
||||
# Return a dummy black image
|
||||
dummy_image = torch.zeros(1, 512, 512, 3)
|
||||
return dummy_image, 512, 512
|
||||
|
||||
def upscale_image(self, api_key, image, upscale_factor, seed):
|
||||
"""Upscale image using Fal AI SeedVR API"""
|
||||
if not api_key or api_key.strip() == "":
|
||||
# Create dummy outputs
|
||||
dummy_image = torch.zeros(1, 512, 512, 3)
|
||||
return (dummy_image, "", 512, 512)
|
||||
|
||||
try:
|
||||
# Clean and set API key FIRST - needed for CDN upload
|
||||
clean_api_key = api_key.strip()
|
||||
key_preview = clean_api_key[:8] + "..." if len(clean_api_key) > 8 else clean_api_key
|
||||
print(f"[Fal SeedVR Upscale] Using API key starting with: {key_preview}")
|
||||
|
||||
# Clear and set the environment variable
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Convert the first image from the batch
|
||||
if image.dim() == 4 and image.shape[0] > 0:
|
||||
first_image = image[0]
|
||||
else:
|
||||
first_image = image
|
||||
|
||||
# Get original dimensions
|
||||
if image.dim() == 4:
|
||||
original_height, original_width = image.shape[1], image.shape[2]
|
||||
else:
|
||||
original_height, original_width = image.shape[0], image.shape[1]
|
||||
|
||||
print(f"[Fal SeedVR Upscale] Original image size: {original_width}x{original_height}")
|
||||
print(f"[Fal SeedVR Upscale] Upscale factor: {upscale_factor}x")
|
||||
print(f"[Fal SeedVR Upscale] Expected output size: {int(original_width * upscale_factor)}x{int(original_height * upscale_factor)}")
|
||||
|
||||
# Upload image to fal.media CDN
|
||||
print(f"[Fal SeedVR Upscale] Uploading image to fal.media CDN...")
|
||||
image_url = self.upload_tensor_to_fal(first_image)
|
||||
|
||||
# Prepare arguments
|
||||
arguments = {
|
||||
"image_url": image_url,
|
||||
"upscale_factor": upscale_factor
|
||||
}
|
||||
|
||||
# Add seed if specified
|
||||
if seed >= 0:
|
||||
arguments["seed"] = seed
|
||||
print(f"[Fal SeedVR Upscale] Using seed: {seed}")
|
||||
|
||||
# Print arguments without exposing potentially large base64 data
|
||||
safe_arguments = {k: v if not (isinstance(v, str) and v.startswith('data:')) else f"<data_uri_{len(v)}_chars>" for k, v in arguments.items()}
|
||||
print(f"[Fal SeedVR Upscale] Making API call with arguments: {safe_arguments}")
|
||||
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Submit the request synchronously
|
||||
print(f"[Fal SeedVR Upscale] Submitting upscale request...")
|
||||
result = fal_client.subscribe(
|
||||
"fal-ai/seedvr/upscale/image",
|
||||
arguments=arguments,
|
||||
with_logs=True
|
||||
)
|
||||
|
||||
print(f"[Fal SeedVR Upscale] API call completed successfully")
|
||||
|
||||
# Extract result
|
||||
if result and "image" in result:
|
||||
image_data = result["image"]
|
||||
image_url = image_data.get("url", "")
|
||||
output_width = image_data.get("width", 0)
|
||||
output_height = image_data.get("height", 0)
|
||||
used_seed = result.get("seed", seed)
|
||||
|
||||
print(f"[Fal SeedVR Upscale] Upscaled image URL: {image_url}")
|
||||
print(f"[Fal SeedVR Upscale] Output dimensions: {output_width}x{output_height}")
|
||||
print(f"[Fal SeedVR Upscale] Used seed: {used_seed}")
|
||||
|
||||
# Download the upscaled image
|
||||
upscaled_tensor, actual_width, actual_height = self.download_image(image_url)
|
||||
|
||||
status_message = f"Successfully upscaled image {upscale_factor}x using Fal AI SeedVR. Output: {actual_width}x{actual_height}"
|
||||
print(f"[Fal SeedVR Upscale] {status_message}")
|
||||
|
||||
return (upscaled_tensor, image_url, actual_width, actual_height)
|
||||
|
||||
else:
|
||||
error_msg = "No image data in API response"
|
||||
print(f"[Fal SeedVR Upscale] {error_msg}")
|
||||
print(f"[Fal SeedVR Upscale] Full response: {result}")
|
||||
# Return dummy outputs
|
||||
dummy_image = torch.zeros(1, 512, 512, 3)
|
||||
return (dummy_image, "", 512, 512)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error in image upscaling: {str(e)}"
|
||||
print(f"[Fal SeedVR Upscale] {error_msg}")
|
||||
|
||||
# Print traceback for debugging
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Return dummy outputs
|
||||
dummy_image = torch.zeros(1, 512, 512, 3)
|
||||
return (dummy_image, "", 512, 512)
|
||||
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FL_Fal_SeedVR_Upscale": FL_Fal_SeedVR_Upscale
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_Fal_SeedVR_Upscale": "FL Fal SeedVR Upscale"
|
||||
}
|
||||
@@ -1,574 +0,0 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import wave
|
||||
|
||||
import fal_client
|
||||
import requests
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
import comfy.model_management
|
||||
from comfy_api.latest import InputImpl, Types, io as comfy_io
|
||||
from server import PromptServer
|
||||
|
||||
|
||||
logger = logging.getLogger("fl_fill_nodes")
|
||||
|
||||
ENDPOINT = "bytedance/seedance-2.0/reference-to-video"
|
||||
EVENT_NAME = "fl_fal_seedance2_progress"
|
||||
|
||||
MAX_IMAGE_BYTES = 30_000_000
|
||||
MAX_AUDIO_BYTES = 15_000_000
|
||||
MAX_VIDEO_BYTES = 50_000_000
|
||||
MIN_VIDEO_PIXELS = 640 * 640
|
||||
MAX_VIDEO_PIXELS = 834 * 1112
|
||||
|
||||
REFERENCE_PATTERN = re.compile(r"@(Image|Video|Audio)(\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _ordered_references(references, prefix):
|
||||
if not references:
|
||||
return []
|
||||
|
||||
indexed = []
|
||||
pattern = re.compile(rf"^{re.escape(prefix)}_(\d+)$")
|
||||
for name, value in references.items():
|
||||
match = pattern.match(name)
|
||||
if match and value is not None:
|
||||
indexed.append((int(match.group(1)), value))
|
||||
return [value for _, value in sorted(indexed)]
|
||||
|
||||
|
||||
def _validate_reference_tags(prompt, image_count, video_count, audio_count):
|
||||
counts = {
|
||||
"image": image_count,
|
||||
"video": video_count,
|
||||
"audio": audio_count,
|
||||
}
|
||||
for kind, raw_index in REFERENCE_PATTERN.findall(prompt):
|
||||
index = int(raw_index)
|
||||
available = counts[kind.lower()]
|
||||
if index < 1 or index > available:
|
||||
raise ValueError(
|
||||
f"Prompt references @{kind.title()}{index}, but only {available} "
|
||||
f"{kind.lower()} reference(s) are connected."
|
||||
)
|
||||
|
||||
|
||||
def _validate_reference_counts(images, videos, audios):
|
||||
if len(images) > 9:
|
||||
raise ValueError("Seedance 2.0 accepts at most 9 reference images.")
|
||||
if len(videos) > 3:
|
||||
raise ValueError("Seedance 2.0 accepts at most 3 reference videos.")
|
||||
if len(audios) > 3:
|
||||
raise ValueError("Seedance 2.0 accepts at most 3 reference audio files.")
|
||||
if len(images) + len(videos) + len(audios) > 12:
|
||||
raise ValueError("Seedance 2.0 accepts at most 12 reference files in total.")
|
||||
if audios and not images and not videos:
|
||||
raise ValueError("Reference audio requires at least one reference image or video.")
|
||||
|
||||
|
||||
def _image_to_png(image, index):
|
||||
if image.ndim != 4 or image.shape[0] != 1:
|
||||
raise ValueError(f"@Image{index} must contain exactly one image.")
|
||||
|
||||
array = image[0].detach().cpu().clamp(0, 1).mul(255).byte().numpy()
|
||||
if array.shape[-1] == 1:
|
||||
array = array[..., 0]
|
||||
elif array.shape[-1] not in (3, 4):
|
||||
raise ValueError(f"@Image{index} has an unsupported channel count: {array.shape[-1]}.")
|
||||
|
||||
buffer = io.BytesIO()
|
||||
Image.fromarray(array).save(buffer, format="PNG")
|
||||
data = buffer.getvalue()
|
||||
if len(data) > MAX_IMAGE_BYTES:
|
||||
raise ValueError(f"@Image{index} is larger than 30 MB after PNG encoding.")
|
||||
return data
|
||||
|
||||
|
||||
def _video_to_mp4(video, index):
|
||||
duration = float(video.get_duration())
|
||||
if duration <= 0:
|
||||
raise ValueError(f"@Video{index} has no playable duration.")
|
||||
|
||||
width, height = video.get_dimensions()
|
||||
pixels = int(width) * int(height)
|
||||
if pixels < MIN_VIDEO_PIXELS or pixels > MAX_VIDEO_PIXELS:
|
||||
raise ValueError(
|
||||
f"@Video{index} is {width}x{height}. Fal requires reference videos between "
|
||||
f"approximately 640x640 and 834x1112 in pixel area."
|
||||
)
|
||||
|
||||
buffer = io.BytesIO()
|
||||
video.save_to(
|
||||
buffer,
|
||||
format=Types.VideoContainer.MP4,
|
||||
codec=Types.VideoCodec.H264,
|
||||
)
|
||||
data = buffer.getvalue()
|
||||
return data, duration
|
||||
|
||||
|
||||
def _audio_to_wav(audio, index):
|
||||
waveform = audio["waveform"]
|
||||
sample_rate = int(audio["sample_rate"])
|
||||
|
||||
if waveform.ndim != 3 or waveform.shape[0] != 1:
|
||||
raise ValueError(f"@Audio{index} must contain exactly one audio clip.")
|
||||
if sample_rate <= 0:
|
||||
raise ValueError(f"@Audio{index} has an invalid sample rate.")
|
||||
|
||||
samples = waveform[0].detach().cpu().float().clamp(-1, 1)
|
||||
if samples.shape[0] < 1:
|
||||
raise ValueError(f"@Audio{index} has no channels.")
|
||||
|
||||
duration = samples.shape[-1] / sample_rate
|
||||
if duration <= 0:
|
||||
raise ValueError(f"@Audio{index} has no playable duration.")
|
||||
|
||||
pcm = (
|
||||
samples.transpose(0, 1)
|
||||
.contiguous()
|
||||
.mul(32767)
|
||||
.round()
|
||||
.to(dtype=torch.int16)
|
||||
.numpy()
|
||||
)
|
||||
buffer = io.BytesIO()
|
||||
with wave.open(buffer, "wb") as wav:
|
||||
wav.setnchannels(samples.shape[0])
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(pcm.tobytes())
|
||||
|
||||
data = buffer.getvalue()
|
||||
if len(data) > MAX_AUDIO_BYTES:
|
||||
raise ValueError(f"@Audio{index} is larger than 15 MB after WAV encoding.")
|
||||
return data, duration
|
||||
|
||||
|
||||
def _prepare_references(images, videos, audios, node_id):
|
||||
prepared_images = []
|
||||
prepared_videos = []
|
||||
prepared_audios = []
|
||||
total_items = len(images) + len(videos) + len(audios)
|
||||
completed = 0
|
||||
|
||||
for index, image in enumerate(images, start=1):
|
||||
_check_interrupted(None, node_id)
|
||||
prepared_images.append(_image_to_png(image, index))
|
||||
completed += 1
|
||||
_emit(node_id, "preparing", value=completed, max=total_items)
|
||||
|
||||
video_duration = 0.0
|
||||
video_bytes = 0
|
||||
for index, video in enumerate(videos, start=1):
|
||||
_check_interrupted(None, node_id)
|
||||
data, duration = _video_to_mp4(video, index)
|
||||
prepared_videos.append(data)
|
||||
video_duration += duration
|
||||
video_bytes += len(data)
|
||||
completed += 1
|
||||
_emit(node_id, "preparing", value=completed, max=total_items)
|
||||
|
||||
if videos and not 2 <= video_duration <= 15:
|
||||
raise ValueError(
|
||||
f"Combined reference video duration is {video_duration:.2f}s; Fal requires 2 to 15 seconds."
|
||||
)
|
||||
if video_bytes >= MAX_VIDEO_BYTES:
|
||||
raise ValueError("Combined reference videos must be smaller than 50 MB after MP4 encoding.")
|
||||
|
||||
audio_duration = 0.0
|
||||
for index, audio in enumerate(audios, start=1):
|
||||
_check_interrupted(None, node_id)
|
||||
data, duration = _audio_to_wav(audio, index)
|
||||
prepared_audios.append(data)
|
||||
audio_duration += duration
|
||||
completed += 1
|
||||
_emit(node_id, "preparing", value=completed, max=total_items)
|
||||
|
||||
if audio_duration > 15:
|
||||
raise ValueError(
|
||||
f"Combined reference audio duration is {audio_duration:.2f}s; Fal allows at most 15 seconds."
|
||||
)
|
||||
|
||||
return prepared_images, prepared_videos, prepared_audios
|
||||
|
||||
|
||||
def _build_arguments(
|
||||
prompt,
|
||||
resolution,
|
||||
duration,
|
||||
aspect_ratio,
|
||||
generate_audio,
|
||||
bitrate_mode,
|
||||
end_user_id,
|
||||
image_urls,
|
||||
video_urls,
|
||||
audio_urls,
|
||||
):
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"resolution": resolution,
|
||||
"duration": duration,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"generate_audio": bool(generate_audio),
|
||||
"bitrate_mode": bitrate_mode,
|
||||
}
|
||||
if image_urls:
|
||||
arguments["image_urls"] = image_urls
|
||||
if video_urls:
|
||||
arguments["video_urls"] = video_urls
|
||||
if audio_urls:
|
||||
arguments["audio_urls"] = audio_urls
|
||||
if end_user_id.strip():
|
||||
arguments["end_user_id"] = end_user_id.strip()
|
||||
return arguments
|
||||
|
||||
|
||||
def _latest_log(status):
|
||||
logs = status.logs or []
|
||||
for entry in reversed(logs):
|
||||
if isinstance(entry, dict):
|
||||
message = entry.get("message")
|
||||
if message:
|
||||
return str(message)[:500]
|
||||
return ""
|
||||
|
||||
|
||||
def _emit(node_id, phase, **details):
|
||||
if PromptServer.instance is None:
|
||||
return
|
||||
payload = {"node": node_id, "phase": phase, **details}
|
||||
try:
|
||||
PromptServer.instance.send_sync(EVENT_NAME, payload)
|
||||
except Exception:
|
||||
logger.debug("Unable to send Seedance 2.0 UI progress event.", exc_info=True)
|
||||
|
||||
|
||||
def _cancel_request(handle):
|
||||
try:
|
||||
handle.cancel()
|
||||
except Exception:
|
||||
logger.debug("Unable to cancel Fal request.", exc_info=True)
|
||||
|
||||
|
||||
def _check_interrupted(handle, node_id):
|
||||
if not comfy.model_management.processing_interrupted():
|
||||
return
|
||||
if handle is not None:
|
||||
_cancel_request(handle)
|
||||
_emit(node_id, "cancelled")
|
||||
comfy.model_management.throw_exception_if_processing_interrupted()
|
||||
|
||||
|
||||
def _upload_references(client, images, videos, audios, node_id):
|
||||
urls = {"images": [], "videos": [], "audios": []}
|
||||
total = len(images) + len(videos) + len(audios)
|
||||
completed = 0
|
||||
|
||||
for index, data in enumerate(images, start=1):
|
||||
_check_interrupted(None, node_id)
|
||||
urls["images"].append(
|
||||
client.upload(data, content_type="image/png", file_name=f"image_{index}.png")
|
||||
)
|
||||
completed += 1
|
||||
_emit(node_id, "uploading", value=completed, max=total)
|
||||
|
||||
for index, data in enumerate(videos, start=1):
|
||||
_check_interrupted(None, node_id)
|
||||
urls["videos"].append(
|
||||
client.upload(data, content_type="video/mp4", file_name=f"video_{index}.mp4")
|
||||
)
|
||||
completed += 1
|
||||
_emit(node_id, "uploading", value=completed, max=total)
|
||||
|
||||
for index, data in enumerate(audios, start=1):
|
||||
_check_interrupted(None, node_id)
|
||||
urls["audios"].append(
|
||||
client.upload(data, content_type="audio/wav", file_name=f"audio_{index}.wav")
|
||||
)
|
||||
completed += 1
|
||||
_emit(node_id, "uploading", value=completed, max=total)
|
||||
|
||||
return urls
|
||||
|
||||
|
||||
def _poll_request(handle, node_id):
|
||||
last_log = ""
|
||||
in_progress_emitted = False
|
||||
status_failures = 0
|
||||
while True:
|
||||
_check_interrupted(handle, node_id)
|
||||
try:
|
||||
status = handle.status(with_logs=True)
|
||||
status_failures = 0
|
||||
except Exception:
|
||||
status_failures += 1
|
||||
if status_failures >= 3:
|
||||
_cancel_request(handle)
|
||||
raise RuntimeError(
|
||||
f"Unable to retrieve Fal status after 3 attempts for request {handle.request_id}."
|
||||
)
|
||||
_emit(
|
||||
node_id,
|
||||
"generating",
|
||||
log=f"Status check failed; retrying ({status_failures}/3)…",
|
||||
request_id=handle.request_id,
|
||||
)
|
||||
time.sleep(2**status_failures)
|
||||
continue
|
||||
if isinstance(status, fal_client.Queued):
|
||||
_emit(
|
||||
node_id,
|
||||
"queued",
|
||||
queue_position=status.position,
|
||||
request_id=handle.request_id,
|
||||
)
|
||||
elif isinstance(status, fal_client.InProgress):
|
||||
latest_log = _latest_log(status)
|
||||
if not in_progress_emitted or latest_log != last_log:
|
||||
in_progress_emitted = True
|
||||
last_log = latest_log
|
||||
_emit(
|
||||
node_id,
|
||||
"generating",
|
||||
log=latest_log,
|
||||
request_id=handle.request_id,
|
||||
)
|
||||
elif isinstance(status, fal_client.Completed):
|
||||
return
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _download_video(url, node_id, handle):
|
||||
buffer = io.BytesIO()
|
||||
with requests.get(url, stream=True, timeout=(10, 300)) as response:
|
||||
response.raise_for_status()
|
||||
total = int(response.headers.get("content-length") or 0)
|
||||
downloaded = 0
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
_check_interrupted(handle, node_id)
|
||||
if not chunk:
|
||||
continue
|
||||
buffer.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
_emit(node_id, "downloading", value=downloaded, max=total)
|
||||
buffer.seek(0)
|
||||
return InputImpl.VideoFromFile(buffer)
|
||||
|
||||
|
||||
def _safe_error(error, api_key):
|
||||
message = str(error)
|
||||
if api_key:
|
||||
message = message.replace(api_key, "***")
|
||||
return message
|
||||
|
||||
|
||||
class FL_Fal_Seedance2_ReferenceToVideo(comfy_io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return comfy_io.Schema(
|
||||
node_id="FL_Fal_Seedance2_ReferenceToVideo",
|
||||
display_name="FL Fal Seedance 2.0 Reference to Video",
|
||||
category="🏵️Fill Nodes/AI",
|
||||
description=(
|
||||
"Generate a Seedance 2.0 video through Fal using prompt, image, video, "
|
||||
"and audio references. Refer to connected assets as @Image1, @Video1, and @Audio1."
|
||||
),
|
||||
inputs=[
|
||||
comfy_io.String.Input(
|
||||
"prompt",
|
||||
multiline=True,
|
||||
dynamic_prompts=True,
|
||||
default="",
|
||||
tooltip=(
|
||||
"Describe the video. Refer to connected assets in order as "
|
||||
"@Image1, @Video1, @Audio1, and so on."
|
||||
),
|
||||
),
|
||||
comfy_io.Autogrow.Input(
|
||||
"reference_images",
|
||||
template=comfy_io.Autogrow.TemplateNames(
|
||||
comfy_io.Image.Input("reference_image"),
|
||||
names=[f"image_{index}" for index in range(1, 10)],
|
||||
min=0,
|
||||
),
|
||||
tooltip="Up to 9 reference images. JPEG, PNG, and WebP are supported by Fal.",
|
||||
),
|
||||
comfy_io.Autogrow.Input(
|
||||
"reference_videos",
|
||||
template=comfy_io.Autogrow.TemplateNames(
|
||||
comfy_io.Video.Input("reference_video"),
|
||||
names=[f"video_{index}" for index in range(1, 4)],
|
||||
min=0,
|
||||
),
|
||||
tooltip="Up to 3 native VIDEO references with 2 to 15 seconds combined duration.",
|
||||
),
|
||||
comfy_io.Autogrow.Input(
|
||||
"reference_audios",
|
||||
template=comfy_io.Autogrow.TemplateNames(
|
||||
comfy_io.Audio.Input("reference_audio"),
|
||||
names=[f"audio_{index}" for index in range(1, 4)],
|
||||
min=0,
|
||||
),
|
||||
tooltip=(
|
||||
"Up to 3 audio references with at most 15 seconds combined duration. "
|
||||
"Audio requires at least one image or video reference."
|
||||
),
|
||||
),
|
||||
comfy_io.Combo.Input(
|
||||
"resolution",
|
||||
options=["480p", "720p", "1080p", "4k"],
|
||||
default="720p",
|
||||
tooltip=(
|
||||
"Output resolution. 4k is present in Fal's current machine-readable schema."
|
||||
),
|
||||
),
|
||||
comfy_io.Combo.Input(
|
||||
"duration",
|
||||
options=["auto", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"],
|
||||
default="auto",
|
||||
tooltip="Output duration in seconds, or auto to let Seedance decide.",
|
||||
),
|
||||
comfy_io.Combo.Input(
|
||||
"aspect_ratio",
|
||||
options=["auto", "21:9", "16:9", "4:3", "1:1", "3:4", "9:16"],
|
||||
default="auto",
|
||||
),
|
||||
comfy_io.Boolean.Input(
|
||||
"generate_audio",
|
||||
default=True,
|
||||
tooltip="Generate synchronized sound effects, ambience, and speech.",
|
||||
),
|
||||
comfy_io.Combo.Input(
|
||||
"bitrate_mode",
|
||||
options=["standard", "high"],
|
||||
default="standard",
|
||||
advanced=True,
|
||||
tooltip="High requests a larger, higher-bitrate output file.",
|
||||
),
|
||||
comfy_io.String.Input(
|
||||
"end_user_id",
|
||||
default="",
|
||||
optional=True,
|
||||
advanced=True,
|
||||
tooltip="Optional end-user identifier sent to Fal.",
|
||||
),
|
||||
comfy_io.String.Input(
|
||||
"fal_api_key",
|
||||
default="",
|
||||
optional=True,
|
||||
advanced=True,
|
||||
tooltip="Optional Fal key override. Leave blank to use the FAL_KEY environment variable.",
|
||||
),
|
||||
],
|
||||
outputs=[
|
||||
comfy_io.Video.Output("video"),
|
||||
comfy_io.String.Output("video_url"),
|
||||
comfy_io.Int.Output("seed"),
|
||||
comfy_io.String.Output("request_id"),
|
||||
],
|
||||
hidden=[comfy_io.Hidden.unique_id],
|
||||
is_api_node=True,
|
||||
not_idempotent=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(
|
||||
cls,
|
||||
prompt,
|
||||
reference_images,
|
||||
reference_videos,
|
||||
reference_audios,
|
||||
resolution,
|
||||
duration,
|
||||
aspect_ratio,
|
||||
generate_audio,
|
||||
bitrate_mode,
|
||||
end_user_id="",
|
||||
fal_api_key="",
|
||||
):
|
||||
node_id = cls.hidden.unique_id if cls.hidden is not None else None
|
||||
api_key = fal_api_key.strip() or os.environ.get("FAL_KEY", "").strip()
|
||||
handle = None
|
||||
|
||||
try:
|
||||
if not prompt.strip():
|
||||
raise ValueError("Prompt cannot be empty.")
|
||||
if not api_key:
|
||||
raise ValueError(
|
||||
"A Fal API key is required. Set FAL_KEY or enter fal_api_key in the node."
|
||||
)
|
||||
|
||||
images = _ordered_references(reference_images, "image")
|
||||
videos = _ordered_references(reference_videos, "video")
|
||||
audios = _ordered_references(reference_audios, "audio")
|
||||
_validate_reference_counts(images, videos, audios)
|
||||
_validate_reference_tags(prompt, len(images), len(videos), len(audios))
|
||||
|
||||
counts = {
|
||||
"images": len(images),
|
||||
"videos": len(videos),
|
||||
"audios": len(audios),
|
||||
}
|
||||
_emit(node_id, "preparing", value=0, max=sum(counts.values()), counts=counts)
|
||||
prepared = _prepare_references(images, videos, audios, node_id)
|
||||
|
||||
client = fal_client.SyncClient(key=api_key)
|
||||
urls = _upload_references(
|
||||
client,
|
||||
prepared[0],
|
||||
prepared[1],
|
||||
prepared[2],
|
||||
node_id,
|
||||
)
|
||||
arguments = _build_arguments(
|
||||
prompt.strip(),
|
||||
resolution,
|
||||
duration,
|
||||
aspect_ratio,
|
||||
generate_audio,
|
||||
bitrate_mode,
|
||||
end_user_id,
|
||||
urls["images"],
|
||||
urls["videos"],
|
||||
urls["audios"],
|
||||
)
|
||||
|
||||
_check_interrupted(None, node_id)
|
||||
handle = client.submit(ENDPOINT, arguments)
|
||||
_emit(node_id, "queued", request_id=handle.request_id)
|
||||
_poll_request(handle, node_id)
|
||||
|
||||
result = handle.get()
|
||||
video_data = result.get("video") if isinstance(result, dict) else None
|
||||
video_url = video_data.get("url") if isinstance(video_data, dict) else None
|
||||
seed = result.get("seed") if isinstance(result, dict) else None
|
||||
if not video_url:
|
||||
raise RuntimeError("Fal completed the request without returning a video URL.")
|
||||
if not isinstance(seed, int):
|
||||
raise RuntimeError("Fal completed the request without returning a seed.")
|
||||
|
||||
video = _download_video(video_url, node_id, handle)
|
||||
_emit(
|
||||
node_id,
|
||||
"complete",
|
||||
video_url=video_url,
|
||||
seed=seed,
|
||||
request_id=handle.request_id,
|
||||
counts=counts,
|
||||
)
|
||||
return comfy_io.NodeOutput(video, video_url, seed, handle.request_id)
|
||||
except Exception as error:
|
||||
message = _safe_error(error, api_key)
|
||||
_emit(
|
||||
node_id,
|
||||
"error",
|
||||
message=message,
|
||||
request_id=handle.request_id if handle is not None else "",
|
||||
)
|
||||
raise RuntimeError(f"Fal Seedance 2.0 failed: {message}") from None
|
||||
@@ -1,253 +0,0 @@
|
||||
import io
|
||||
import numpy as np
|
||||
import os
|
||||
import tempfile
|
||||
import requests
|
||||
from PIL import Image
|
||||
import torch
|
||||
import cv2
|
||||
from tqdm import tqdm
|
||||
import fal_client
|
||||
|
||||
class FL_Fal_Seedance_i2v:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {
|
||||
"multiline": False,
|
||||
"default": "",
|
||||
"placeholder": "Enter your Fal AI API key"
|
||||
}),
|
||||
"image": ("IMAGE",),
|
||||
"prompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"default": "",
|
||||
"placeholder": "Describe the video animation you want to create"
|
||||
}),
|
||||
"cut_prompt_1": ("STRING",),
|
||||
"cut_prompt_2": ("STRING",),
|
||||
"cut_prompt_3": ("STRING",),
|
||||
"cut_prompt_4": ("STRING",),
|
||||
"resolution": (["480p", "720p", "1080p"], {
|
||||
"default": "720p"
|
||||
}),
|
||||
"duration": ("INT", {
|
||||
"default": 6,
|
||||
"min": 3,
|
||||
"max": 12,
|
||||
"step": 1
|
||||
}),
|
||||
"camera_fixed": ("BOOLEAN", {
|
||||
"default": True
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"default": -1,
|
||||
"min": -1,
|
||||
"max": 999999999
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames", "video_url", "status_message")
|
||||
FUNCTION = "generate_video"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def upload_tensor_to_fal(self, tensor_image):
|
||||
"""Upload tensor image to fal.media CDN and return the URL."""
|
||||
# Convert tensor to PIL Image
|
||||
if tensor_image.dim() == 4:
|
||||
tensor_image = tensor_image.squeeze(0)
|
||||
if tensor_image.shape[0] == 3: # CHW to HWC
|
||||
tensor_image = tensor_image.permute(1, 2, 0)
|
||||
|
||||
# Ensure values are in [0, 1] range and convert to [0, 255]
|
||||
if tensor_image.max() <= 1.0:
|
||||
tensor_image = (tensor_image * 255).clamp(0, 255)
|
||||
|
||||
# Convert to uint8 and then to PIL Image
|
||||
np_image = tensor_image.cpu().numpy().astype(np.uint8)
|
||||
pil_image = Image.fromarray(np_image)
|
||||
|
||||
# Upload to fal.media CDN
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
url = fal_client.upload(image_bytes, content_type="image/png")
|
||||
return url
|
||||
|
||||
def download_video_frames(self, video_url):
|
||||
"""Download video and extract all frames"""
|
||||
try:
|
||||
# Create temporary directory
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
video_path = os.path.join(temp_dir, "video.mp4")
|
||||
|
||||
# Download video with progress bar
|
||||
response = requests.get(video_url, stream=True)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
|
||||
with open(video_path, 'wb') as f:
|
||||
with tqdm(total=total_size, unit='iB', unit_scale=True, desc="Downloading video") as pbar:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
|
||||
# Extract frames using OpenCV
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
|
||||
with tqdm(total=total_frames, desc="Extracting frames") as pbar:
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
# Extract every frame
|
||||
# Convert BGR to RGB
|
||||
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Convert to PIL Image
|
||||
pil_image = Image.fromarray(frame_rgb)
|
||||
|
||||
# Convert to tensor
|
||||
tensor_frame = torch.from_numpy(np.array(pil_image)).float() / 255.0
|
||||
tensor_frame = tensor_frame.unsqueeze(0) # Add batch dimension
|
||||
|
||||
frames.append(tensor_frame)
|
||||
|
||||
frame_count += 1
|
||||
pbar.update(1)
|
||||
|
||||
cap.release()
|
||||
|
||||
if frames:
|
||||
# Stack all frames
|
||||
all_frames = torch.cat(frames, dim=0)
|
||||
return all_frames
|
||||
else:
|
||||
# Return a dummy black frame if no frames extracted
|
||||
dummy_frame = torch.zeros(1, 512, 512, 3)
|
||||
return dummy_frame
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error downloading/processing video: {str(e)}")
|
||||
# Return a dummy black frame
|
||||
dummy_frame = torch.zeros(1, 512, 512, 3)
|
||||
return dummy_frame
|
||||
|
||||
def generate_video(self, api_key, image, prompt, resolution, duration, camera_fixed, seed,
|
||||
cut_prompt_1="", cut_prompt_2="", cut_prompt_3="", cut_prompt_4=""):
|
||||
if not api_key or api_key.strip() == "":
|
||||
# Create dummy outputs
|
||||
dummy_frames = torch.zeros(1, 512, 512, 3)
|
||||
return (dummy_frames, "", "Error: API key is required")
|
||||
|
||||
try:
|
||||
# Clean and set API key FIRST - needed for CDN upload
|
||||
clean_api_key = api_key.strip()
|
||||
key_preview = clean_api_key[:8] + "..." if len(clean_api_key) > 8 else clean_api_key
|
||||
print(f"[Fal Seedance] Using API key starting with: {key_preview}")
|
||||
|
||||
# Clear and set the environment variable
|
||||
if "FAL_KEY" in os.environ:
|
||||
del os.environ["FAL_KEY"]
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Convert the first image from the batch and upload to CDN
|
||||
if image.dim() == 4 and image.shape[0] > 0:
|
||||
first_image = image[0]
|
||||
else:
|
||||
first_image = image
|
||||
|
||||
print(f"[Fal Seedance] Uploading image to fal.media CDN...")
|
||||
image_url = self.upload_tensor_to_fal(first_image)
|
||||
print(f"[Fal Seedance] Uploaded image to CDN: {image_url[:80]}...")
|
||||
|
||||
# Build the complete prompt with cut prompts
|
||||
complete_prompt = prompt.strip()
|
||||
|
||||
# Add cut prompts if provided
|
||||
cut_prompts = [cut_prompt_1, cut_prompt_2, cut_prompt_3, cut_prompt_4]
|
||||
for i, cut_prompt in enumerate(cut_prompts):
|
||||
if cut_prompt and cut_prompt.strip():
|
||||
complete_prompt += f" [cut] {cut_prompt.strip()}"
|
||||
|
||||
print(f"[Fal Seedance] Complete prompt: {complete_prompt}")
|
||||
|
||||
# Prepare arguments
|
||||
arguments = {
|
||||
"image_url": image_url,
|
||||
"prompt": complete_prompt,
|
||||
"resolution": resolution,
|
||||
"duration": duration,
|
||||
"camera_fixed": camera_fixed
|
||||
}
|
||||
|
||||
# Add seed if specified
|
||||
if seed >= 0:
|
||||
arguments["seed"] = seed
|
||||
|
||||
# Print arguments without exposing potentially large base64 data
|
||||
safe_arguments = {k: v if not (isinstance(v, str) and v.startswith('data:')) else f"<data_uri_{len(v)}_chars>" for k, v in arguments.items()}
|
||||
print(f"[Fal Seedance] Making API call with arguments: {safe_arguments}")
|
||||
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Submit the request synchronously
|
||||
result = fal_client.subscribe(
|
||||
"fal-ai/bytedance/seedance/v1/pro/image-to-video",
|
||||
arguments=arguments,
|
||||
with_logs=True
|
||||
)
|
||||
|
||||
print(f"[Fal Seedance] API call completed successfully")
|
||||
|
||||
if result and "video" in result and "url" in result["video"]:
|
||||
video_url = result["video"]["url"]
|
||||
|
||||
# Download and extract frames
|
||||
all_frames = self.download_video_frames(video_url)
|
||||
|
||||
total_frames = all_frames.shape[0]
|
||||
status_message = f"Successfully generated video using Fal AI Bytedance Seedance. Extracted {total_frames} frames."
|
||||
|
||||
return (all_frames, video_url, status_message)
|
||||
|
||||
else:
|
||||
error_msg = "No video URL in API response"
|
||||
print(f"[Fal Seedance] {error_msg}")
|
||||
# Return dummy outputs
|
||||
dummy_frames = torch.zeros(1, 512, 512, 3)
|
||||
return (dummy_frames, "", error_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error in video generation: {str(e)}"
|
||||
print(f"[Fal Seedance] {error_msg}")
|
||||
|
||||
# Return dummy outputs
|
||||
dummy_frames = torch.zeros(1, 512, 512, 3)
|
||||
return (dummy_frames, "", error_msg)
|
||||
|
||||
# Node registration
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FL_Fal_Seedance_i2v": FL_Fal_Seedance_i2v
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_Fal_Seedance_i2v": "FL Fal Seedance i2v"
|
||||
}
|
||||
@@ -1,496 +0,0 @@
|
||||
# FL_Fal_Seedream_Edit: Fal AI ByteDance Seedream v4 Edit API Node
|
||||
import os
|
||||
import uuid
|
||||
import json
|
||||
import time
|
||||
import io
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
import fal_client
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import random
|
||||
from typing import Tuple, List, Dict, Union, Optional
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
|
||||
class FL_Fal_Seedream_Edit:
|
||||
"""
|
||||
A ComfyUI node for the Fal AI ByteDance Seedream v4 Edit API.
|
||||
Takes multiple images and a prompt to edit them using Seedream's capabilities.
|
||||
"""
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "STRING", "STRING")
|
||||
RETURN_NAMES = ("images", "image_urls", "seed", "status_msg")
|
||||
FUNCTION = "edit_images"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False,
|
||||
"description": "Fal AI API key"}),
|
||||
"prompt": ("STRING", {"default": "Dress the model in the clothes and shoes.",
|
||||
"multiline": True, "forceInput": True,
|
||||
"description": "The prompt for image editing"}),
|
||||
"image_size": (["square_hd", "square", "portrait_4_3", "portrait_16_9", "landscape_4_3", "landscape_16_9"],
|
||||
{"default": "square_hd", "description": "The size of the generated image"}),
|
||||
"num_images": ("INT", {"default": 1, "min": 1, "max": 4, "step": 1,
|
||||
"description": "Number of images to generate"}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647,
|
||||
"description": "Random seed for image generation (0 = random)"}),
|
||||
"sync_mode": ("BOOLEAN", {"default": False,
|
||||
"description": "Wait for image generation before returning"}),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}),
|
||||
},
|
||||
"optional": {
|
||||
"image_1": ("IMAGE", {"description": "First input image to edit"}),
|
||||
"image_2": ("IMAGE", {"description": "Second input image to edit"}),
|
||||
"image_3": ("IMAGE", {"description": "Third input image to edit"}),
|
||||
"image_4": ("IMAGE", {"description": "Fourth input image to edit"}),
|
||||
"image_5": ("IMAGE", {"description": "Fifth input image to edit"}),
|
||||
"image_6": ("IMAGE", {"description": "Sixth input image to edit"}),
|
||||
"image_7": ("IMAGE", {"description": "Seventh input image to edit"}),
|
||||
"image_8": ("IMAGE", {"description": "Eighth input image to edit"}),
|
||||
"image_9": ("IMAGE", {"description": "Ninth input image to edit"}),
|
||||
"image_10": ("IMAGE", {"description": "Tenth input image to edit"}),
|
||||
"retry_indefinitely": ("BOOLEAN", {"default": False}),
|
||||
"use_custom_resolution": ("BOOLEAN", {"default": False,
|
||||
"description": "Override preset image size with custom width/height"}),
|
||||
"auto_scale_to_minimum": ("BOOLEAN", {"default": True,
|
||||
"description": "Auto-scale dimensions below 1024 to meet minimum while preserving aspect ratio"}),
|
||||
"custom_width": ("INT", {"default": 1280, "min": 256, "max": 4096, "step": 8,
|
||||
"description": "Custom width (auto-scaled to meet API minimums if needed)"}),
|
||||
"custom_height": ("INT", {"default": 1280, "min": 256, "max": 4096, "step": 8,
|
||||
"description": "Custom height (auto-scaled to meet API minimums if needed)"}),
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_Fal_Seedream_Edit] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
if hasattr(self, 'log_messages'):
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _create_error_image(self, error_message="API Error", width=1024, height=1024):
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
font = None
|
||||
try:
|
||||
font_options = ['arial.ttf', 'DejaVuSans.ttf', 'FreeSans.ttf', 'NotoSans-Regular.ttf']
|
||||
for font_name in font_options:
|
||||
try:
|
||||
font = ImageFont.truetype(font_name, 24)
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
if font is None:
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Calculate text position (centered)
|
||||
try:
|
||||
text_bbox = draw.textbbox((0,0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
except AttributeError:
|
||||
text_width, text_height = draw.textsize(error_message, font=font)
|
||||
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log(f"Created error image: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _process_tensor_to_pil_list(self, tensor_image: Optional[torch.Tensor], image_name_prefix: str = "Image") -> Optional[List[Image.Image]]:
|
||||
if tensor_image is None:
|
||||
self._log(f"{image_name_prefix} input is None, skipping PIL conversion.")
|
||||
return None
|
||||
if not isinstance(tensor_image, torch.Tensor):
|
||||
self._log(f"{image_name_prefix} is not a tensor (type: {type(tensor_image)}), skipping.")
|
||||
return None
|
||||
|
||||
pil_images = []
|
||||
if tensor_image.ndim == 4: # Batch of images (B, H, W, C)
|
||||
if tensor_image.shape[0] == 0:
|
||||
self._log(f"{image_name_prefix} batch is empty (shape: {tensor_image.shape}).")
|
||||
return None
|
||||
for i in range(tensor_image.shape[0]):
|
||||
img_np = tensor_image[i].cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted {image_name_prefix} batch item {i} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
elif tensor_image.ndim == 3: # Single image (H, W, C)
|
||||
img_np = tensor_image.cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted single {image_name_prefix} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
else:
|
||||
self._log(f"Cannot convert {image_name_prefix} with ndim {tensor_image.ndim} (shape: {tensor_image.shape}) to PIL Image(s).")
|
||||
return None
|
||||
|
||||
return pil_images if pil_images else None
|
||||
|
||||
def _upload_image_to_fal(self, pil_image: Image.Image) -> str:
|
||||
"""Upload PIL image to fal.media CDN and return the URL."""
|
||||
try:
|
||||
# Convert RGBA to RGB if necessary (PNG handles it but let's be safe)
|
||||
if pil_image.mode == 'RGBA':
|
||||
background = Image.new('RGB', pil_image.size, (255, 255, 255))
|
||||
background.paste(pil_image, mask=pil_image.split()[-1])
|
||||
pil_image = background
|
||||
self._log(f"Converted RGBA image to RGB with white background")
|
||||
elif pil_image.mode != 'RGB':
|
||||
pil_image = pil_image.convert('RGB')
|
||||
self._log(f"Converted image mode to RGB")
|
||||
|
||||
# Upload to fal.media CDN
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format="PNG")
|
||||
image_bytes = buffered.getvalue()
|
||||
url = fal_client.upload(image_bytes, content_type="image/png")
|
||||
self._log(f"Uploaded image to fal CDN: {url[:80]}...")
|
||||
return url
|
||||
except Exception as e:
|
||||
self._log(f"Error uploading image to fal CDN: {str(e)}")
|
||||
raise
|
||||
|
||||
async def _edit_images_async(self, api_key, prompt, input_images, image_size, num_images, seed, sync_mode, max_retries, retry_indefinitely, use_custom_resolution, auto_scale_to_minimum, custom_width, custom_height):
|
||||
try:
|
||||
# Calculate seed
|
||||
actual_seed = seed if seed != 0 else random.randint(1, 2147483647)
|
||||
|
||||
self._log(f"Starting image editing with seed {actual_seed} and prompt: '{prompt[:50]}...'")
|
||||
|
||||
# Set API key FIRST - needed for CDN upload
|
||||
clean_api_key = api_key.strip()
|
||||
os.environ["FAL_KEY"] = clean_api_key
|
||||
|
||||
# Upload images to fal.media CDN
|
||||
image_urls = []
|
||||
if input_images:
|
||||
self._log(f"Uploading {len(input_images)} images to fal.media CDN...")
|
||||
for i, pil_image in enumerate(input_images):
|
||||
try:
|
||||
img_url = self._upload_image_to_fal(pil_image)
|
||||
image_urls.append(img_url)
|
||||
self._log(f"Successfully uploaded image {i+1}/{len(input_images)} to CDN")
|
||||
except Exception as e:
|
||||
self._log(f"Error uploading image {i+1} to CDN: {str(e)}")
|
||||
error_msg = f"Error: Failed to upload image {i+1}: {str(e)}"
|
||||
return self._create_error_image(error_msg), "", str(actual_seed), error_msg
|
||||
else:
|
||||
error_msg = "Error: No images provided for editing"
|
||||
return self._create_error_image(error_msg), "", str(actual_seed), error_msg
|
||||
|
||||
# Prepare image size parameter
|
||||
image_size_param = image_size
|
||||
# Check if custom dimensions are requested and valid
|
||||
if use_custom_resolution:
|
||||
if custom_width and custom_height and custom_width >= 1024 and custom_height >= 1024:
|
||||
image_size_param = {
|
||||
"width": custom_width,
|
||||
"height": custom_height
|
||||
}
|
||||
self._log(f"Using custom image size: {custom_width}x{custom_height}")
|
||||
elif custom_width and custom_height:
|
||||
# Custom dimensions provided but below minimum
|
||||
aspect_ratio = custom_width / custom_height
|
||||
self._log(f"Custom dimensions {custom_width}x{custom_height} below minimum. Aspect ratio: {aspect_ratio:.2f}")
|
||||
|
||||
if auto_scale_to_minimum:
|
||||
# Scale up dimensions to meet minimum while preserving aspect ratio
|
||||
min_dimension = min(custom_width, custom_height)
|
||||
if min_dimension < 1024:
|
||||
scale_factor = 1024 / min_dimension
|
||||
scaled_width = int(custom_width * scale_factor)
|
||||
scaled_height = int(custom_height * scale_factor)
|
||||
|
||||
# Ensure we don't exceed maximum dimensions
|
||||
if scaled_width > 4096 or scaled_height > 4096:
|
||||
max_scale = min(4096 / custom_width, 4096 / custom_height)
|
||||
scaled_width = int(custom_width * max_scale)
|
||||
scaled_height = int(custom_height * max_scale)
|
||||
|
||||
# Round to nearest 64 pixels for better compatibility
|
||||
scaled_width = (scaled_width // 64) * 64
|
||||
scaled_height = (scaled_height // 64) * 64
|
||||
|
||||
# Ensure minimums after rounding
|
||||
scaled_width = max(1024, scaled_width)
|
||||
scaled_height = max(1024, scaled_height)
|
||||
|
||||
image_size_param = {
|
||||
"width": scaled_width,
|
||||
"height": scaled_height
|
||||
}
|
||||
self._log(f"Auto-scaled from {custom_width}x{custom_height} to {scaled_width}x{scaled_height} preserving aspect ratio")
|
||||
else:
|
||||
# Map aspect ratios to presets (with tolerance)
|
||||
if aspect_ratio >= 2.1: # ~21:9 ratio
|
||||
image_size_param = "landscape_16_9" # Closest available wide option
|
||||
self._log(f"Selected landscape_16_9 based on wide aspect ratio")
|
||||
elif aspect_ratio >= 1.6: # ~16:9 ratio
|
||||
image_size_param = "landscape_16_9"
|
||||
self._log(f"Selected landscape_16_9 based on 16:9 aspect ratio")
|
||||
elif aspect_ratio >= 1.2: # ~4:3 ratio
|
||||
image_size_param = "landscape_4_3"
|
||||
self._log(f"Selected landscape_4_3 based on 4:3 aspect ratio")
|
||||
elif aspect_ratio >= 0.9: # ~1:1 ratio
|
||||
image_size_param = "square_hd" if "hd" in image_size else "square"
|
||||
self._log(f"Selected square based on 1:1 aspect ratio")
|
||||
elif aspect_ratio >= 0.7: # ~3:4 ratio
|
||||
image_size_param = "portrait_4_3"
|
||||
self._log(f"Selected portrait_4_3 based on 3:4 aspect ratio")
|
||||
elif aspect_ratio >= 0.5: # ~9:16 ratio
|
||||
image_size_param = "portrait_16_9"
|
||||
self._log(f"Selected portrait_16_9 based on 9:16 aspect ratio")
|
||||
else: # Very tall
|
||||
image_size_param = "portrait_16_9"
|
||||
self._log(f"Selected portrait_16_9 based on tall aspect ratio")
|
||||
else:
|
||||
self._log("Custom resolution requested but dimensions are invalid or missing. Using preset image size.")
|
||||
image_size_param = image_size
|
||||
self._log(f"Using preset image size: {image_size}")
|
||||
else:
|
||||
self._log(f"Using preset image size: {image_size}")
|
||||
|
||||
# Prepare the arguments for fal_client
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"image_urls": image_urls,
|
||||
"image_size": image_size_param,
|
||||
"num_images": num_images,
|
||||
"seed": actual_seed,
|
||||
"sync_mode": sync_mode
|
||||
}
|
||||
|
||||
self._log(f"Calling Fal AI Seedream Edit API with {len(image_urls)} images...")
|
||||
|
||||
# Define a callback for queue updates
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
self._log(f"API Log: {log['message']}")
|
||||
|
||||
# Make the API call in executor to avoid blocking
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def make_fal_call():
|
||||
try:
|
||||
# Force reload the fal_client module to avoid caching issues
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
import fal_client
|
||||
|
||||
# Make the API call using fal_client.subscribe
|
||||
result = fal_client.subscribe(
|
||||
"fal-ai/bytedance/seedream/v4/edit",
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
self._log(f"API call error: {str(e)}")
|
||||
return None
|
||||
|
||||
result = await loop.run_in_executor(None, make_fal_call)
|
||||
|
||||
if result is None:
|
||||
error_msg = "Error: API call failed"
|
||||
return self._create_error_image(error_msg), "", str(actual_seed), error_msg
|
||||
|
||||
self._log("API call completed successfully")
|
||||
|
||||
# Extract image URLs and seed from the result
|
||||
output_image_urls = []
|
||||
returned_seed = actual_seed
|
||||
|
||||
if "images" in result and len(result["images"]) > 0:
|
||||
for img_info in result["images"]:
|
||||
if "url" in img_info:
|
||||
output_image_urls.append(img_info["url"])
|
||||
self._log(f"Found {len(output_image_urls)} edited images in response")
|
||||
else:
|
||||
self._log("Warning: No images found in result")
|
||||
error_msg = "Error: No images in API response"
|
||||
return self._create_error_image(error_msg), "", str(actual_seed), error_msg
|
||||
|
||||
# Extract returned seed
|
||||
if "seed" in result:
|
||||
returned_seed = result["seed"]
|
||||
self._log(f"API returned seed: {returned_seed}")
|
||||
|
||||
# Download and process all generated images
|
||||
try:
|
||||
self._log(f"Downloading {len(output_image_urls)} edited images...")
|
||||
|
||||
processed_images = []
|
||||
url_list = []
|
||||
|
||||
for i, image_url in enumerate(output_image_urls):
|
||||
# Download image
|
||||
dl_response = requests.get(image_url)
|
||||
dl_response.raise_for_status()
|
||||
|
||||
# Convert to PIL Image
|
||||
pil_image = Image.open(io.BytesIO(dl_response.content))
|
||||
|
||||
# Convert to RGB if necessary
|
||||
if pil_image.mode != 'RGB':
|
||||
pil_image = pil_image.convert('RGB')
|
||||
|
||||
# Convert to numpy array
|
||||
np_image = np.array(pil_image).astype(np.float32) / 255.0
|
||||
|
||||
# Convert to tensor (add batch dimension)
|
||||
image_tensor = torch.from_numpy(np_image).unsqueeze(0)
|
||||
processed_images.append(image_tensor)
|
||||
url_list.append(image_url)
|
||||
|
||||
self._log(f"Processed image {i+1}/{len(output_image_urls)} with shape {image_tensor.shape}")
|
||||
|
||||
# Concatenate all images
|
||||
if len(processed_images) > 1:
|
||||
# Handle multiple images - ensure same dimensions
|
||||
max_height = max(img.shape[1] for img in processed_images)
|
||||
max_width = max(img.shape[2] for img in processed_images)
|
||||
|
||||
resized_images = []
|
||||
for img_tensor in processed_images:
|
||||
current_h, current_w = img_tensor.shape[1], img_tensor.shape[2]
|
||||
|
||||
if current_h == max_height and current_w == max_width:
|
||||
resized_images.append(img_tensor)
|
||||
else:
|
||||
# Pad the image
|
||||
pad_h = max_height - current_h
|
||||
pad_w = max_width - current_w
|
||||
pad_left = pad_w // 2
|
||||
pad_right = pad_w - pad_left
|
||||
pad_top = pad_h // 2
|
||||
pad_bottom = pad_h - pad_top
|
||||
|
||||
padded_img = torch.nn.functional.pad(
|
||||
img_tensor.permute(0, 3, 1, 2),
|
||||
(pad_left, pad_right, pad_top, pad_bottom),
|
||||
mode='constant',
|
||||
value=0
|
||||
).permute(0, 2, 3, 1)
|
||||
|
||||
resized_images.append(padded_img)
|
||||
|
||||
combined_tensor = torch.cat(resized_images, dim=0)
|
||||
else:
|
||||
combined_tensor = processed_images[0]
|
||||
|
||||
combined_urls = " | ".join(url_list)
|
||||
|
||||
self._log(f"Successfully processed {len(processed_images)} images with final shape {combined_tensor.shape}")
|
||||
|
||||
return combined_tensor, combined_urls, str(returned_seed), f"Success: {len(processed_images)} images edited successfully"
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Download Error: {str(e)}"
|
||||
self._log(error_msg)
|
||||
return self._create_error_image(error_msg), "", str(actual_seed), error_msg
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error in async image editing: {str(e)}")
|
||||
error_msg = f"Error: {str(e)}"
|
||||
return self._create_error_image(error_msg), "", str(actual_seed), error_msg
|
||||
|
||||
def edit_images(self, api_key, prompt, image_size="square_hd", num_images=1, seed=0, sync_mode=False, max_retries=3,
|
||||
image_1=None, image_2=None, image_3=None, image_4=None, image_5=None,
|
||||
image_6=None, image_7=None, image_8=None, image_9=None, image_10=None,
|
||||
retry_indefinitely=False, use_custom_resolution=False, auto_scale_to_minimum=True,
|
||||
custom_width=1280, custom_height=1280, **kwargs):
|
||||
self.log_messages = []
|
||||
if not api_key:
|
||||
error_msg = "API key not provided."
|
||||
self._log(error_msg)
|
||||
error_img_instance = self._create_error_image(error_msg)
|
||||
return (error_img_instance, "", "0", error_msg)
|
||||
|
||||
# Collect all input images (up to 10 as per API limit)
|
||||
input_images = []
|
||||
input_tensors = [image_1, image_2, image_3, image_4, image_5,
|
||||
image_6, image_7, image_8, image_9, image_10]
|
||||
|
||||
for i, tensor in enumerate(input_tensors):
|
||||
if tensor is not None:
|
||||
pil_images = self._process_tensor_to_pil_list(tensor, f"Image{i+1}")
|
||||
if pil_images:
|
||||
input_images.extend(pil_images)
|
||||
|
||||
# Limit to 10 images as per API specification
|
||||
if len(input_images) >= 10:
|
||||
input_images = input_images[:10]
|
||||
self._log("Limiting to 10 images as per API specification")
|
||||
break
|
||||
|
||||
if not input_images:
|
||||
error_msg = "No valid input images provided."
|
||||
self._log(error_msg)
|
||||
error_img_instance = self._create_error_image(error_msg)
|
||||
return (error_img_instance, "", "0", error_msg)
|
||||
|
||||
self._log(f"Processing {len(input_images)} input images for editing")
|
||||
|
||||
# Run async processing using thread pool to avoid event loop conflicts
|
||||
def run_sync_edit():
|
||||
"""Run async edit in a new thread with its own event loop"""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(self._edit_images_async(
|
||||
api_key, prompt, input_images, image_size, num_images, seed,
|
||||
sync_mode, max_retries, retry_indefinitely, use_custom_resolution,
|
||||
auto_scale_to_minimum, custom_width, custom_height
|
||||
))
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
result = None
|
||||
try:
|
||||
# Use thread pool executor to run async code in separate thread
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(run_sync_edit)
|
||||
result = future.result(timeout=300) # 5 minute timeout
|
||||
except concurrent.futures.TimeoutError:
|
||||
self._log("Processing timed out after 5 minutes")
|
||||
error_img = self._create_error_image("Processing timeout")
|
||||
return (error_img, "", "0", "Processing timed out after 5 minutes")
|
||||
except Exception as e:
|
||||
self._log(f"Error in processing: {str(e)}")
|
||||
error_img = self._create_error_image(f"Processing error: {str(e)}")
|
||||
return (error_img, "", "0", f"Processing error: {str(e)}")
|
||||
|
||||
if result is None:
|
||||
error_img = self._create_error_image("Processing failed to produce results")
|
||||
return (error_img, "", "0", "Processing failed to produce results")
|
||||
|
||||
# Extract results
|
||||
images, image_urls, returned_seed, status_msg = result
|
||||
|
||||
# Combine log messages with status
|
||||
final_log_output = "Processing Logs:\n" + "\n".join(self.log_messages) + "\n\n" + status_msg
|
||||
|
||||
return (images, image_urls, returned_seed, final_log_output)
|
||||
@@ -1,324 +0,0 @@
|
||||
import os
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import folder_paths
|
||||
import tempfile
|
||||
import importlib
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
import cv2
|
||||
import torchaudio
|
||||
import subprocess
|
||||
|
||||
class FL_Fal_Sora:
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"fal_api_key": ("STRING", {"default": ""}),
|
||||
"prompt": ("STRING", {"multiline": True, "default": "A dramatic scene..."}),
|
||||
"use_pro": ("BOOLEAN", {"default": False}),
|
||||
"resolution": (["auto", "720p", "1080p"], {"default": "auto"}),
|
||||
"aspect_ratio": (["auto", "16:9", "9:16"], {"default": "16:9"}),
|
||||
"duration": ([4, 8, 12], {"default": 4}),
|
||||
"openai_api_key": ("STRING", {"default": ""}),
|
||||
"nth_frame": ("INT", {"default": 1, "min": 1, "max": 4, "step": 1}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "AUDIO", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames", "audio", "video_url", "status_msg")
|
||||
FUNCTION = "generate_video"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def generate_video(self, fal_api_key, prompt, use_pro, resolution, aspect_ratio, duration, openai_api_key, nth_frame, image=None):
|
||||
# Helper function for empty audio
|
||||
def empty_audio():
|
||||
return {"waveform": torch.zeros((1, 1, 0)), "sample_rate": 44100}
|
||||
|
||||
try:
|
||||
# Validate API key
|
||||
if not fal_api_key or fal_api_key.strip() == "":
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), "", "❌ Error: Fal API key is required")
|
||||
|
||||
# Validate prompt
|
||||
if not prompt or prompt.strip() == "":
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), "", "❌ Error: Prompt is required")
|
||||
|
||||
# Clear and set FAL_KEY environment variable
|
||||
if 'FAL_KEY' in os.environ:
|
||||
del os.environ['FAL_KEY']
|
||||
os.environ['FAL_KEY'] = fal_api_key
|
||||
|
||||
# Reload fal_client to ensure it picks up the new API key
|
||||
import sys
|
||||
if 'fal_client' in sys.modules:
|
||||
del sys.modules['fal_client']
|
||||
|
||||
import fal_client
|
||||
|
||||
# Determine which endpoint to use
|
||||
is_image_to_video = image is not None
|
||||
original_resolution = resolution
|
||||
original_aspect_ratio = aspect_ratio
|
||||
|
||||
if is_image_to_video:
|
||||
if use_pro:
|
||||
endpoint = "fal-ai/sora-2/image-to-video/pro"
|
||||
valid_resolutions = ["auto", "720p", "1080p"]
|
||||
else:
|
||||
endpoint = "fal-ai/sora-2/image-to-video"
|
||||
valid_resolutions = ["auto", "720p"]
|
||||
else:
|
||||
if use_pro:
|
||||
endpoint = "fal-ai/sora-2/text-to-video/pro"
|
||||
valid_resolutions = ["720p", "1080p"]
|
||||
# For text-to-video, default to 720p if "auto" is selected
|
||||
if resolution == "auto":
|
||||
resolution = "720p"
|
||||
print("⚠️ 'auto' resolution not supported for text-to-video, using '720p'")
|
||||
else:
|
||||
endpoint = "fal-ai/sora-2/text-to-video"
|
||||
valid_resolutions = ["720p"]
|
||||
if resolution != "720p":
|
||||
print(f"⚠️ Resolution '{resolution}' not available for non-PRO text-to-video, using '720p'")
|
||||
resolution = "720p"
|
||||
|
||||
# Validate resolution
|
||||
if resolution not in valid_resolutions:
|
||||
old_res = resolution
|
||||
resolution = valid_resolutions[0]
|
||||
print(f"⚠️ Resolution '{old_res}' not valid for {endpoint}, using '{resolution}'")
|
||||
print(f" Valid resolutions: {', '.join(valid_resolutions)}")
|
||||
|
||||
# Validate aspect_ratio for text-to-video (no auto option)
|
||||
if not is_image_to_video and aspect_ratio == "auto":
|
||||
aspect_ratio = "16:9"
|
||||
print("⚠️ 'auto' aspect ratio not supported for text-to-video, using '16:9'")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"📹 Using endpoint: {endpoint}")
|
||||
print(f"📝 Prompt: {prompt[:100]}...")
|
||||
print(f"🎬 Settings: {resolution}, {aspect_ratio}, {duration}s")
|
||||
if original_resolution != resolution or original_aspect_ratio != aspect_ratio:
|
||||
print(f"ℹ️ Note: Some parameters were auto-corrected for this endpoint")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# Build arguments
|
||||
arguments = {
|
||||
"prompt": prompt,
|
||||
"resolution": resolution,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
"duration": duration,
|
||||
}
|
||||
|
||||
# Add OpenAI API key if provided (to avoid billing from Fal)
|
||||
if openai_api_key and openai_api_key.strip():
|
||||
arguments["api_key"] = openai_api_key
|
||||
print("🔑 Using OpenAI API key (billed to your OpenAI account)")
|
||||
|
||||
# Handle image input for image-to-video
|
||||
image_url = None
|
||||
temp_image_path = None
|
||||
if is_image_to_video:
|
||||
print("🖼️ Image provided - using image-to-video mode")
|
||||
|
||||
# Convert tensor to PIL Image
|
||||
i = 255. * image[0].cpu().numpy()
|
||||
img_pil = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
|
||||
|
||||
# Save to temporary file
|
||||
temp_image_path = tempfile.NamedTemporaryFile(delete=False, suffix='.png').name
|
||||
img_pil.save(temp_image_path)
|
||||
|
||||
# Upload image to Fal
|
||||
print("📤 Uploading image to Fal...")
|
||||
image_url = fal_client.upload_file(temp_image_path)
|
||||
arguments["image_url"] = image_url
|
||||
print(f"✅ Image uploaded: {image_url}")
|
||||
|
||||
# Call the API
|
||||
print(f"🚀 Calling Fal API: {endpoint}")
|
||||
|
||||
def on_queue_update(update):
|
||||
if isinstance(update, fal_client.InProgress):
|
||||
for log in update.logs:
|
||||
print(f"📝 {log['message']}")
|
||||
|
||||
result = fal_client.subscribe(
|
||||
endpoint,
|
||||
arguments=arguments,
|
||||
with_logs=True,
|
||||
on_queue_update=on_queue_update,
|
||||
)
|
||||
|
||||
# Clean up temp image file
|
||||
if temp_image_path and os.path.exists(temp_image_path):
|
||||
os.remove(temp_image_path)
|
||||
|
||||
# Get video URL from result
|
||||
if not result:
|
||||
error_msg = "❌ API returned empty result"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), "", error_msg)
|
||||
|
||||
if 'video' not in result:
|
||||
error_msg = f"❌ No 'video' field in API response. Response keys: {list(result.keys())}"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), "", error_msg)
|
||||
|
||||
if 'url' not in result['video']:
|
||||
error_msg = f"❌ No 'url' in video response. Video keys: {list(result['video'].keys())}"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), "", error_msg)
|
||||
|
||||
video_url = result['video']['url']
|
||||
print(f"✅ Video generated: {video_url}")
|
||||
|
||||
# Download the video
|
||||
print("⬇️ Downloading video...")
|
||||
response = requests.get(video_url, stream=True)
|
||||
|
||||
if response.status_code != 200:
|
||||
error_msg = f"❌ Failed to download video. HTTP Status: {response.status_code}"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), video_url, error_msg)
|
||||
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
|
||||
temp_video_path = tempfile.NamedTemporaryFile(delete=False, suffix='.mp4').name
|
||||
|
||||
with open(temp_video_path, 'wb') as f, tqdm(
|
||||
desc="Downloading",
|
||||
total=total_size,
|
||||
unit='iB',
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
) as pbar:
|
||||
for data in response.iter_content(chunk_size=1024):
|
||||
size = f.write(data)
|
||||
pbar.update(size)
|
||||
|
||||
print(f"✅ Video downloaded to: {temp_video_path}")
|
||||
|
||||
# Extract audio from video using ffmpeg
|
||||
print("🎵 Extracting audio from video...")
|
||||
temp_audio_path = tempfile.NamedTemporaryFile(delete=False, suffix='.wav').name
|
||||
|
||||
try:
|
||||
ffmpeg_cmd = [
|
||||
'ffmpeg',
|
||||
'-i', temp_video_path,
|
||||
'-vn', # No video
|
||||
'-acodec', 'pcm_s16le', # PCM 16-bit little-endian
|
||||
'-ar', '44100', # 44.1kHz sample rate
|
||||
'-ac', '2', # Stereo
|
||||
'-y', # Overwrite output file
|
||||
temp_audio_path
|
||||
]
|
||||
|
||||
result_ffmpeg = subprocess.run(ffmpeg_cmd, capture_output=True, text=True)
|
||||
|
||||
if result_ffmpeg.returncode != 0:
|
||||
print(f"⚠️ FFmpeg warning: {result_ffmpeg.stderr}")
|
||||
print("⚠️ Video may not contain audio, using empty audio")
|
||||
audio_output = empty_audio()
|
||||
else:
|
||||
# Load audio using torchaudio
|
||||
waveform, sample_rate = torchaudio.load(temp_audio_path)
|
||||
audio_output = {
|
||||
"waveform": waveform.unsqueeze(0), # Add batch dimension
|
||||
"sample_rate": sample_rate
|
||||
}
|
||||
print(f"✅ Audio extracted: {waveform.shape[1]} samples at {sample_rate}Hz")
|
||||
|
||||
# Clean up temp audio file
|
||||
if os.path.exists(temp_audio_path):
|
||||
os.remove(temp_audio_path)
|
||||
|
||||
except FileNotFoundError:
|
||||
print("⚠️ FFmpeg not found. Install ffmpeg to extract audio. Using empty audio.")
|
||||
audio_output = empty_audio()
|
||||
except Exception as e:
|
||||
print(f"⚠️ Error extracting audio: {str(e)}. Using empty audio.")
|
||||
audio_output = empty_audio()
|
||||
|
||||
# Extract frames from video
|
||||
print(f"🎞️ Extracting frames (every {nth_frame} frame(s))...")
|
||||
cap = cv2.VideoCapture(temp_video_path)
|
||||
|
||||
if not cap.isOpened():
|
||||
error_msg = f"❌ Failed to open video file: {temp_video_path}"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), audio_output, video_url, error_msg)
|
||||
|
||||
frames = []
|
||||
frame_count = 0
|
||||
|
||||
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
|
||||
if total_frames == 0:
|
||||
cap.release()
|
||||
error_msg = "❌ Video file contains 0 frames"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), audio_output, video_url, error_msg)
|
||||
|
||||
with tqdm(total=total_frames, desc="Extracting frames") as pbar:
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
if frame_count % nth_frame == 0:
|
||||
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
frames.append(frame_rgb)
|
||||
|
||||
frame_count += 1
|
||||
pbar.update(1)
|
||||
|
||||
cap.release()
|
||||
|
||||
# Clean up temp video file
|
||||
if os.path.exists(temp_video_path):
|
||||
os.remove(temp_video_path)
|
||||
|
||||
if not frames:
|
||||
error_msg = "❌ No frames extracted from video"
|
||||
print(error_msg)
|
||||
return (torch.zeros(1, 64, 64, 3), audio_output, video_url, error_msg)
|
||||
|
||||
print(f"✅ Extracted {len(frames)} frames")
|
||||
|
||||
# Convert frames to tensor
|
||||
frames_np = np.array(frames).astype(np.float32) / 255.0
|
||||
frames_tensor = torch.from_numpy(frames_np)
|
||||
|
||||
success_msg = f"✅ Successfully generated video with {len(frames)} frames using {endpoint}"
|
||||
print(success_msg)
|
||||
|
||||
return (frames_tensor, audio_output, video_url, success_msg)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"❌ Error: {str(e)}"
|
||||
# Remove large base64 data URIs from error message for readability
|
||||
if "data:image" in error_msg or "data:audio" in error_msg:
|
||||
error_msg = error_msg[:500] + "... (truncated)"
|
||||
print(error_msg)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return (torch.zeros(1, 64, 64, 3), empty_audio(), "", error_msg)
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FL_Fal_Sora": FL_Fal_Sora
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_Fal_Sora": "FL Fal Sora 2"
|
||||
}
|
||||
@@ -1,548 +0,0 @@
|
||||
import os
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
import requests
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
import time
|
||||
import traceback
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import random
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
|
||||
class FL_GeminiImageEditor:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt": ("STRING", {"multiline": True}),
|
||||
"api_key": ("STRING", {"default": "", "multiline": False}),
|
||||
"model": (["models/gemini-2.0-flash-exp", "models/gemini-2.0-flash-preview-image-generation", "models/gemini-2.5-flash-image-preview", "models/gemini-2.5-flash-image", "models/gemini-3-pro-image-preview"], {"default": "models/gemini-2.5-flash-image"}),
|
||||
"aspect_ratio": (["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"], {"default": "1:1"}),
|
||||
"image_size": (["1K", "2K", "4K"], {"default": "1K", "tooltip": "Resolution size (2K/4K only supported by gemini-3-pro-image-preview)"}),
|
||||
"always_square": ("BOOLEAN", {"default": False, "description": "When enabled, pads images to square dimensions. When disabled, outputs original resolution as image list."}),
|
||||
"temperature": ("FLOAT", {"default": 1, "min": 0.0, "max": 2.0, "step": 0.05}),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}),
|
||||
"batch_size": ("INT", {"default": 1, "min": 1, "max": 8, "step": 1}),
|
||||
},
|
||||
"optional": {
|
||||
"seed": ("INT", {"default": 66666666, "min": 0, "max": 66666666}),
|
||||
"image1": ("IMAGE",),
|
||||
"image2": ("IMAGE",),
|
||||
"image3": ("IMAGE",),
|
||||
"image4": ("IMAGE",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("image", "API Respond")
|
||||
OUTPUT_IS_LIST = (True, False)
|
||||
FUNCTION = "generate_image"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize logging system"""
|
||||
self.log_messages = [] # Global log message storage
|
||||
self.min_size = 1024 # Minimum size for both width and height
|
||||
|
||||
# Check google-genai version
|
||||
try:
|
||||
import importlib.metadata
|
||||
genai_version = importlib.metadata.version('google-genai')
|
||||
self._log(f"Current google-genai version: {genai_version}")
|
||||
|
||||
# Check if version meets minimum requirements
|
||||
from packaging import version
|
||||
if version.parse(genai_version) < version.parse('0.8.0'):
|
||||
self._log("Warning: google-genai version is too low, recommend upgrading to the latest version")
|
||||
self._log("Suggested: pip install -q -U google-genai")
|
||||
except Exception as e:
|
||||
self._log(f"Unable to check google-genai version: {e}")
|
||||
|
||||
def _log(self, message):
|
||||
"""Global logging function: record to log list"""
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_GeminiImageGenerator] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
if hasattr(self, 'log_messages'):
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _pad_image_to_minimum_size(self, pil_image):
|
||||
"""Pad image with white to ensure it's at least min_size x min_size while preserving aspect ratio"""
|
||||
width, height = pil_image.size
|
||||
|
||||
# If already meets minimum size requirements, return unchanged
|
||||
if width >= self.min_size and height >= self.min_size:
|
||||
return pil_image
|
||||
|
||||
# Calculate new dimensions (preserve aspect ratio)
|
||||
new_width = max(width, self.min_size)
|
||||
new_height = max(height, self.min_size)
|
||||
|
||||
# Create new white canvas
|
||||
new_image = Image.new('RGB', (new_width, new_height), color=(255, 255, 255))
|
||||
|
||||
# Calculate position to paste the original image (centered)
|
||||
paste_x = (new_width - width) // 2
|
||||
paste_y = (new_height - height) // 2
|
||||
|
||||
# Paste original image onto the white canvas
|
||||
new_image.paste(pil_image, (paste_x, paste_y))
|
||||
|
||||
self._log(f"Padded image from {width}x{height} to {new_width}x{new_height} with white borders")
|
||||
return new_image
|
||||
|
||||
def _create_error_image(self, error_message="API Failed to return an image", width=1024, height=1024):
|
||||
"""Create black image with error text"""
|
||||
# Create black image
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
# Try to use a system font
|
||||
try:
|
||||
# Try to find a font that exists on most systems
|
||||
font_options = ['Arial.ttf', 'DejaVuSans.ttf', 'FreeSans.ttf', 'NotoSans-Regular.ttf']
|
||||
font = None
|
||||
|
||||
for font_name in font_options:
|
||||
try:
|
||||
font = ImageFont.truetype(font_name, 24)
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
|
||||
if font is None:
|
||||
# Fall back to default font
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
# If everything fails, use default
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Calculate text position (centered)
|
||||
text_width = draw.textlength(error_message, font=font) if hasattr(draw, 'textlength') else \
|
||||
font.getsize(error_message)[0]
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = height / 2 - 12 # Vertically centered
|
||||
|
||||
# Draw text
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
|
||||
# Convert to tensor format [1, H, W, 3]
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
|
||||
self._log(f"Created error image with message: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _generate_empty_image(self, width=1024, height=1024):
|
||||
"""Generate standard format empty RGB image tensor - ensure ComfyUI compatible format [B,H,W,C]"""
|
||||
# Now just use error image with default message
|
||||
return self._create_error_image(width=width, height=height)
|
||||
|
||||
def _process_tensor_to_pil(self, tensor, name="Image"):
|
||||
"""Convert a tensor to a PIL image for API submission"""
|
||||
try:
|
||||
if tensor is None:
|
||||
self._log(f"{name} is None, skipping")
|
||||
return None
|
||||
|
||||
# Ensure tensor is in correct format [1, H, W, 3]
|
||||
if len(tensor.shape) == 4 and tensor.shape[0] == 1:
|
||||
# Get first frame image
|
||||
image_np = tensor[0].cpu().numpy()
|
||||
|
||||
# Convert to uint8 format for PIL
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
|
||||
# Create PIL image
|
||||
pil_image = Image.fromarray(image_np)
|
||||
|
||||
self._log(f"{name} processed successfully, size: {pil_image.width}x{pil_image.height}")
|
||||
return pil_image
|
||||
else:
|
||||
self._log(f"{name} format incorrect: {tensor.shape}")
|
||||
return None
|
||||
except Exception as e:
|
||||
self._log(f"Error processing {name}: {str(e)}")
|
||||
return None
|
||||
|
||||
def _call_gemini_api(self, client, model, contents, gen_config, retry_count=0, max_retries=3, batch_id=0):
|
||||
"""Call Gemini API with retry logic"""
|
||||
try:
|
||||
self._log(f"[Batch {batch_id}] API call attempt #{retry_count + 1}")
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=contents,
|
||||
config=gen_config
|
||||
)
|
||||
|
||||
# Validate response structure
|
||||
if not hasattr(response, 'candidates') or not response.candidates:
|
||||
self._log(f"[Batch {batch_id}] Empty response: No candidates found")
|
||||
if retry_count < max_retries - 1:
|
||||
self._log(f"[Batch {batch_id}] Retrying in 2 seconds... (Attempt {retry_count + 1}/{max_retries})")
|
||||
time.sleep(2) # Wait 2 seconds before retry
|
||||
return self._call_gemini_api(client, model, contents, gen_config,
|
||||
retry_count + 1, max_retries, batch_id)
|
||||
else:
|
||||
self._log(f"[Batch {batch_id}] Maximum retries ({max_retries}) reached. Returning empty response.")
|
||||
return None
|
||||
|
||||
# Check if candidates[0].content exists
|
||||
if not hasattr(response.candidates[0], 'content') or response.candidates[0].content is None:
|
||||
self._log(f"[Batch {batch_id}] Invalid response: candidates[0].content is missing")
|
||||
if retry_count < max_retries - 1:
|
||||
self._log(f"[Batch {batch_id}] Retrying in 2 seconds... (Attempt {retry_count + 1}/{max_retries})")
|
||||
time.sleep(2)
|
||||
return self._call_gemini_api(client, model, contents, gen_config,
|
||||
retry_count + 1, max_retries, batch_id)
|
||||
else:
|
||||
self._log(f"[Batch {batch_id}] Maximum retries ({max_retries}) reached. Returning empty response.")
|
||||
return None
|
||||
|
||||
# Check if content.parts exists
|
||||
if not hasattr(response.candidates[0].content, 'parts') or response.candidates[0].content.parts is None:
|
||||
self._log(f"[Batch {batch_id}] Invalid response: candidates[0].content.parts is missing")
|
||||
if retry_count < max_retries - 1:
|
||||
self._log(f"[Batch {batch_id}] Retrying in 2 seconds... (Attempt {retry_count + 1}/{max_retries})")
|
||||
time.sleep(2)
|
||||
return self._call_gemini_api(client, model, contents, gen_config,
|
||||
retry_count + 1, max_retries, batch_id)
|
||||
else:
|
||||
self._log(f"[Batch {batch_id}] Maximum retries ({max_retries}) reached. Returning empty response.")
|
||||
return None
|
||||
|
||||
# Valid response, return it
|
||||
self._log(f"[Batch {batch_id}] Valid API response received")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"[Batch {batch_id}] API call error: {str(e)}")
|
||||
if retry_count < max_retries - 1:
|
||||
wait_time = 2 * (retry_count + 1) # Progressive backoff: 2s, 4s, 6s...
|
||||
self._log(
|
||||
f"[Batch {batch_id}] Retrying in {wait_time} seconds... (Attempt {retry_count + 1}/{max_retries})")
|
||||
time.sleep(wait_time)
|
||||
return self._call_gemini_api(client, model, contents, gen_config,
|
||||
retry_count + 1, max_retries, batch_id)
|
||||
else:
|
||||
self._log(f"[Batch {batch_id}] Maximum retries ({max_retries}) reached. Giving up.")
|
||||
return None
|
||||
|
||||
def _process_api_response(self, response, batch_id=0, always_square=False):
|
||||
"""Process API response and extract image tensor"""
|
||||
if response is None:
|
||||
self._log(f"[Batch {batch_id}] No valid response to process")
|
||||
error_msg = "API Failed to return an image"
|
||||
return self._create_error_image(error_msg), error_msg
|
||||
|
||||
response_text = ""
|
||||
|
||||
# Check if response contains valid data
|
||||
if not hasattr(response, 'candidates') or not response.candidates:
|
||||
self._log(f"[Batch {batch_id}] No candidates in API response")
|
||||
error_msg = "API returned an empty response"
|
||||
return self._create_error_image(error_msg), error_msg
|
||||
|
||||
# Iterate through response parts
|
||||
for part in response.candidates[0].content.parts:
|
||||
# Check if it's a text part
|
||||
if hasattr(part, 'text') and part.text is not None:
|
||||
text_content = part.text
|
||||
response_text += text_content
|
||||
self._log(
|
||||
f"[Batch {batch_id}] API returned text: {text_content[:100]}..." if len(
|
||||
text_content) > 100 else text_content)
|
||||
|
||||
# Check if it's an image part
|
||||
elif hasattr(part, 'inline_data') and part.inline_data is not None:
|
||||
self._log(f"[Batch {batch_id}] API returned image data")
|
||||
try:
|
||||
# Get image data
|
||||
image_data = part.inline_data.data
|
||||
mime_type = part.inline_data.mime_type if hasattr(part.inline_data, 'mime_type') else "unknown"
|
||||
|
||||
# Confirm data is not empty
|
||||
if not image_data or len(image_data) < 100:
|
||||
self._log(f"[Batch {batch_id}] Warning: Image data is empty or too small")
|
||||
continue
|
||||
|
||||
# Multiple methods to try opening the image
|
||||
pil_image = None
|
||||
|
||||
# Method 1: Direct PIL open
|
||||
try:
|
||||
pil_image = Image.open(BytesIO(image_data))
|
||||
self._log(
|
||||
f"[Batch {batch_id}] Direct PIL open successful, size: {pil_image.width}x{pil_image.height}")
|
||||
except Exception as e1:
|
||||
self._log(f"[Batch {batch_id}] Direct PIL open failed: {str(e1)}")
|
||||
|
||||
# Method 2: Save to temp file and open
|
||||
try:
|
||||
temp_file = os.path.join(tempfile.gettempdir(),
|
||||
f"gemini_image_{batch_id}_{int(time.time())}.png")
|
||||
with open(temp_file, "wb") as f:
|
||||
f.write(image_data)
|
||||
|
||||
pil_image = Image.open(temp_file)
|
||||
self._log(f"[Batch {batch_id}] Opening via temp file successful")
|
||||
except Exception as e2:
|
||||
self._log(f"[Batch {batch_id}] Opening via temp file failed: {str(e2)}")
|
||||
|
||||
# Try more methods if needed
|
||||
# Additional opening methods from original code could be added here if necessary
|
||||
|
||||
# Ensure image loaded successfully
|
||||
if pil_image is None:
|
||||
self._log(f"[Batch {batch_id}] Cannot open image, skipping")
|
||||
continue
|
||||
|
||||
# Ensure image is RGB mode
|
||||
if pil_image.mode != 'RGB':
|
||||
pil_image = pil_image.convert('RGB')
|
||||
self._log(f"[Batch {batch_id}] Image converted to RGB mode")
|
||||
|
||||
# Store original dimensions for logging
|
||||
width, height = pil_image.size
|
||||
self._log(f"[Batch {batch_id}] Original image size: {width}x{height}")
|
||||
|
||||
# Apply padding if always_square is enabled and image needs it
|
||||
if always_square and (width < self.min_size or height < self.min_size):
|
||||
self._log(
|
||||
f"[Batch {batch_id}] Image size {width}x{height} is smaller than minimum {self.min_size}x{self.min_size}, padding needed")
|
||||
pil_image = self._pad_image_to_minimum_size(pil_image)
|
||||
elif always_square:
|
||||
self._log(f"[Batch {batch_id}] Always square enabled but image already meets minimum size")
|
||||
else:
|
||||
self._log(f"[Batch {batch_id}] Always square disabled, keeping original size: {width}x{height}")
|
||||
|
||||
# Convert to ComfyUI format
|
||||
img_array = np.array(pil_image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
|
||||
self._log(f"[Batch {batch_id}] Image converted to tensor successfully, shape: {img_tensor.shape}")
|
||||
return img_tensor, response_text
|
||||
except Exception as e:
|
||||
self._log(f"[Batch {batch_id}] Image processing error: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
# If we got here, no image was found
|
||||
self._log(f"[Batch {batch_id}] No image data found in API response")
|
||||
error_msg = "API Failed to return an image"
|
||||
return self._create_error_image(error_msg), response_text if response_text else error_msg
|
||||
|
||||
async def _generate_single_image_async(self, prompt, api_key, model, temperature, max_retries,
|
||||
batch_id, seed, reference_images, always_square=False,
|
||||
aspect_ratio="1:1", image_size="1K"):
|
||||
"""Generate a single image asynchronously for batch processing"""
|
||||
try:
|
||||
# Create client instance - each batch gets its own client
|
||||
client = genai.Client(api_key=api_key)
|
||||
|
||||
# Use provided seed or generate random one
|
||||
actual_seed = seed if seed != 0 else random.randint(1, 0xffffff)
|
||||
self._log(f"[Batch {batch_id}] Using seed: {actual_seed}")
|
||||
|
||||
# Configure generation parameters
|
||||
gen_config_params = {
|
||||
"temperature": temperature,
|
||||
"seed": actual_seed,
|
||||
"response_modalities": ['Text', 'Image']
|
||||
}
|
||||
|
||||
# Add image config if supported
|
||||
try:
|
||||
# Check if ImageConfig supports imageSize parameter (SDK >= 1.50)
|
||||
import inspect
|
||||
image_config_sig = inspect.signature(types.ImageConfig)
|
||||
supports_image_size = 'imageSize' in image_config_sig.parameters or 'image_size' in image_config_sig.parameters
|
||||
|
||||
image_config_params = {"aspectRatio": aspect_ratio}
|
||||
# Only add imageSize for gemini-3-pro model if SDK supports it
|
||||
if "gemini-3-pro" in model and supports_image_size:
|
||||
image_config_params["imageSize"] = image_size
|
||||
self._log(f"[Batch {batch_id}] Using imageSize: {image_size}")
|
||||
elif "gemini-3-pro" in model:
|
||||
self._log(f"[Batch {batch_id}] Warning: imageSize not supported in this SDK version. Upgrade google-genai to 1.50+ for 4K support")
|
||||
|
||||
gen_config_params["image_config"] = types.ImageConfig(**image_config_params)
|
||||
except Exception as e:
|
||||
self._log(f"[Batch {batch_id}] ImageConfig not available: {e}, using basic config")
|
||||
|
||||
gen_config = types.GenerateContentConfig(**gen_config_params)
|
||||
|
||||
# Create content parts
|
||||
content_parts = []
|
||||
|
||||
# Add prompt
|
||||
simple_prompt = f"Create a detailed image of: {prompt}"
|
||||
content_parts.append(simple_prompt)
|
||||
|
||||
# Add reference images if provided
|
||||
for img in reference_images:
|
||||
if img is not None:
|
||||
content_parts.append(img)
|
||||
|
||||
# Make API call with synchronous method (will run in thread pool)
|
||||
loop = asyncio.get_event_loop()
|
||||
response = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._call_gemini_api(
|
||||
client=client,
|
||||
model=model,
|
||||
contents=content_parts,
|
||||
gen_config=gen_config,
|
||||
max_retries=max_retries,
|
||||
batch_id=batch_id
|
||||
)
|
||||
)
|
||||
|
||||
# Process the response and return the image tensor and text
|
||||
img_tensor, response_text = await loop.run_in_executor(
|
||||
None, lambda: self._process_api_response(response, batch_id, always_square)
|
||||
)
|
||||
|
||||
# If processing failed, return the error image
|
||||
if img_tensor is None:
|
||||
error_msg = f"Batch {batch_id}: API Failed to return an image"
|
||||
return self._create_error_image(error_msg), error_msg, batch_id
|
||||
|
||||
return img_tensor, response_text, batch_id
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"[Batch {batch_id}] Error in async image generation: {str(e)}")
|
||||
error_msg = f"Batch {batch_id}: Error: {str(e)}"
|
||||
return self._create_error_image(error_msg), error_msg, batch_id
|
||||
|
||||
def generate_image(self, prompt, api_key, model, aspect_ratio, image_size, temperature, max_retries=3, batch_size=1,
|
||||
seed=66666666, always_square=False, image1=None, image2=None, image3=None, image4=None):
|
||||
"""Generate batch of images with parallel API calls"""
|
||||
# Reset log messages
|
||||
self.log_messages = []
|
||||
all_response_text = ""
|
||||
|
||||
try:
|
||||
# Check if API key is provided
|
||||
if not api_key:
|
||||
error_message = "Error: No API key provided. Please enter Google API key in the node."
|
||||
self._log(error_message)
|
||||
error_img = self._create_error_image("API key required")
|
||||
full_text = "## Error\n" + error_message + "\n\n## Instructions\n1. Enter your Google API key in the node"
|
||||
|
||||
# Create list of error images for all batch sizes
|
||||
error_imgs = [error_img] * batch_size
|
||||
return (error_imgs, full_text)
|
||||
|
||||
self._log(f"Starting batch generation of {batch_size} images")
|
||||
|
||||
# Process reference images once
|
||||
reference_pil_images = []
|
||||
image_tensors = [image1, image2, image3, image4]
|
||||
|
||||
for i, img_tensor in enumerate(image_tensors):
|
||||
if img_tensor is not None:
|
||||
pil_img = self._process_tensor_to_pil(img_tensor, f"Reference Image {i + 1}")
|
||||
if pil_img:
|
||||
reference_pil_images.append(pil_img)
|
||||
self._log(f"Added reference image {i + 1} to batch processing")
|
||||
|
||||
# Setup async tasks for each batch item
|
||||
async def run_batch():
|
||||
tasks = []
|
||||
|
||||
# Create tasks for each batch item
|
||||
for i in range(batch_size):
|
||||
# If seed is specified (non-zero), increment it for each batch item
|
||||
# Otherwise each batch will use a random seed
|
||||
batch_seed = seed + i if seed != 0 else 0
|
||||
|
||||
task = self._generate_single_image_async(
|
||||
prompt=prompt,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
max_retries=max_retries,
|
||||
batch_id=i + 1,
|
||||
seed=batch_seed,
|
||||
reference_images=reference_pil_images,
|
||||
always_square=always_square,
|
||||
aspect_ratio=aspect_ratio,
|
||||
image_size=image_size
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
# Run all tasks concurrently
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
# Run the async batch processing using thread pool to avoid event loop conflicts
|
||||
def run_sync_batch():
|
||||
"""Run async batch in a new thread with its own event loop"""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(run_batch())
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
results = None # Initialize results
|
||||
try:
|
||||
# Use thread pool executor to run async code in separate thread
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(run_sync_batch)
|
||||
results = future.result(timeout=300) # 5 minute timeout
|
||||
except concurrent.futures.TimeoutError:
|
||||
self._log("Async processing timed out after 5 minutes")
|
||||
error_imgs = [self._create_error_image("Processing timeout")] * batch_size
|
||||
return (error_imgs, "Processing timed out after 5 minutes")
|
||||
except Exception as e:
|
||||
self._log(f"Error in async processing: {str(e)}")
|
||||
traceback.print_exc()
|
||||
# Create list of error images
|
||||
error_imgs = [self._create_error_image(f"Async processing error: {str(e)}")] * batch_size
|
||||
return (error_imgs, f"Async processing error: {str(e)}")
|
||||
|
||||
# Process results (ensure results is not None if an error occurred before assignment)
|
||||
if results is None:
|
||||
self._log("Async processing did not yield results, possibly due to an earlier error before gather.")
|
||||
error_imgs = [self._create_error_image("Async processing failed to produce results")] * batch_size
|
||||
return (error_imgs, "Async processing failed to produce results")
|
||||
|
||||
# Process results
|
||||
all_tensors = []
|
||||
batch_texts = []
|
||||
|
||||
for img_tensor, text, batch_id in results:
|
||||
# Always add tensor to list since we now always have a valid tensor
|
||||
# (either real image or error image)
|
||||
all_tensors.append(img_tensor)
|
||||
batch_texts.append(f"## Batch {batch_id} Response\n{text}")
|
||||
|
||||
self._log(f"Successfully created list of {len(all_tensors)} images")
|
||||
|
||||
# Combine all texts
|
||||
all_response_text = "## Batch Processing Results\n" + "\n".join(self.log_messages) + "\n\n" + "\n\n".join(
|
||||
batch_texts)
|
||||
|
||||
return (all_tensors, all_response_text)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Error during batch processing: {str(e)}"
|
||||
self._log(error_message)
|
||||
traceback.print_exc()
|
||||
|
||||
# Create list of error images
|
||||
error_imgs = [self._create_error_image(f"Error: {str(e)}")] * batch_size
|
||||
|
||||
# Combine logs and error info
|
||||
full_text = "## Processing Log\n" + "\n".join(self.log_messages) + "\n\n## Error\n" + error_message
|
||||
return (error_imgs, full_text)
|
||||
@@ -1,472 +0,0 @@
|
||||
import os
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
import requests
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
import time
|
||||
import traceback
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import random
|
||||
from typing import List, Tuple, Optional
|
||||
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
# Assuming ImageBatch is still needed if we are batching results, or can be removed if Gemini returns a batch
|
||||
# from nodes import ImageBatch
|
||||
|
||||
class FL_GeminiImageGenADV:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"inputcount": ("INT", {"default": 1, "min": 1, "max": 100, "step": 1}),
|
||||
"api_key": ("STRING", {"default": os.getenv("GEMINI_API_KEY", ""), "multiline": False}),
|
||||
"model": (["models/gemini-2.0-flash-exp", "models/gemini-2.0-flash-preview-image-generation", "models/gemini-2.5-flash-image-preview", "models/gemini-2.5-flash-image", "models/gemini-3-pro-image-preview"], {"default": "models/gemini-2.5-flash-image"}),
|
||||
"aspect_ratio": (["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"], {"default": "1:1"}),
|
||||
"image_size": (["1K", "2K", "4K"], {"default": "1K", "tooltip": "Resolution size (2K/4K only supported by gemini-3-pro-image-preview)"}),
|
||||
"always_square": ("BOOLEAN", {"default": False, "description": "When enabled, pads images to square dimensions. When disabled, outputs original resolution as image list."}),
|
||||
"temperature": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.05}),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 5, "step": 1}),
|
||||
"prompt_1": ("STRING", {"multiline": False, "default": "Describe image 1", "forceInput": True}),
|
||||
},
|
||||
"optional": {
|
||||
"image_1": ("IMAGE", {}), # Moved image_1 to optional. Default will be None if not connected.
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffff}), # Restored full seed range
|
||||
"retry_indefinitely": ("BOOLEAN", {"default": False}),
|
||||
# Subsequent image_i and prompt_i will be handled by **kwargs based on inputcount
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("images", "API_responses")
|
||||
OUTPUT_IS_LIST = (True, False)
|
||||
FUNCTION = "generate_images_advanced"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
DESCRIPTION = """
|
||||
Generates images using Gemini based on multiple image/prompt pairs.
|
||||
Each pair triggers an asynchronous API call. Results are batched.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
self.min_size = 1024 # Minimum size from Editor
|
||||
try:
|
||||
import importlib.metadata
|
||||
genai_version = importlib.metadata.version('google-genai')
|
||||
self._log(f"Current google-genai version: {genai_version}")
|
||||
from packaging import version # Ensure packaging is imported
|
||||
if version.parse(genai_version) < version.parse('0.8.0'): # Example version, check Gemini docs
|
||||
self._log("Warning: google-genai version is too low, recommend upgrading to the latest version.")
|
||||
self._log("Suggested: pip install -q -U google-genai")
|
||||
except Exception as e:
|
||||
self._log(f"Unable to check google-genai version: {e}")
|
||||
|
||||
def _log(self, message):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_GeminiImageGenADV] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
if hasattr(self, 'log_messages'):
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _pad_image_to_minimum_size(self, pil_image):
|
||||
width, height = pil_image.size
|
||||
if width >= self.min_size and height >= self.min_size:
|
||||
return pil_image
|
||||
new_width = max(width, self.min_size)
|
||||
new_height = max(height, self.min_size)
|
||||
new_image = Image.new('RGB', (new_width, new_height), color=(255, 255, 255)) # White padding from Editor
|
||||
paste_x = (new_width - width) // 2
|
||||
paste_y = (new_height - height) // 2
|
||||
new_image.paste(pil_image, (paste_x, paste_y))
|
||||
self._log(f"Padded image from {width}x{height} to {new_width}x{new_height} with white borders")
|
||||
return new_image
|
||||
|
||||
def _create_error_image(self, error_message="API Error", width=1024, height=1024): # Default size from Editor
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0)) # Black error image from Editor
|
||||
draw = ImageDraw.Draw(image)
|
||||
font = None
|
||||
try:
|
||||
# Try to find a font that exists on most systems
|
||||
font_options = ['arial.ttf', 'DejaVuSans.ttf', 'FreeSans.ttf', 'NotoSans-Regular.ttf']
|
||||
for font_name in font_options:
|
||||
try:
|
||||
font = ImageFont.truetype(font_name, 24) # Font size from Editor
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
if font is None:
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Calculate text position (centered)
|
||||
try: # Newer PIL versions
|
||||
text_bbox = draw.textbbox((0,0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
except AttributeError: # Older PIL versions
|
||||
text_width, text_height = draw.textsize(error_message, font=font)
|
||||
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font) # Red text from Editor
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log(f"Created error image: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _process_tensor_to_pil_list(self, tensor_image: Optional[torch.Tensor], image_name_prefix: str = "Image") -> Optional[List[Image.Image]]:
|
||||
if tensor_image is None:
|
||||
self._log(f"{image_name_prefix} input is None, skipping PIL conversion.")
|
||||
return None
|
||||
if not isinstance(tensor_image, torch.Tensor):
|
||||
self._log(f"{image_name_prefix} is not a tensor (type: {type(tensor_image)}), skipping.")
|
||||
return None
|
||||
|
||||
pil_images = []
|
||||
if tensor_image.ndim == 4: # Batch of images (B, H, W, C)
|
||||
if tensor_image.shape[0] == 0:
|
||||
self._log(f"{image_name_prefix} batch is empty (shape: {tensor_image.shape}).")
|
||||
return None
|
||||
for i in range(tensor_image.shape[0]):
|
||||
img_np = tensor_image[i].cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted {image_name_prefix} batch item {i} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
elif tensor_image.ndim == 3: # Single image (H, W, C)
|
||||
img_np = tensor_image.cpu().numpy()
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
self._log(f"Converted single {image_name_prefix} (original shape: {tensor_image.shape}) to PIL Image (size: {pil_image.size}).")
|
||||
pil_images.append(pil_image)
|
||||
else:
|
||||
self._log(f"Cannot convert {image_name_prefix} with ndim {tensor_image.ndim} (shape: {tensor_image.shape}) to PIL Image(s).")
|
||||
return None
|
||||
|
||||
return pil_images if pil_images else None
|
||||
|
||||
def _call_gemini_api(self, client_instance, model_name_full, contents, gen_config_obj, retry_indefinitely, retry_count=0, max_retries=3, call_id="0"):
|
||||
try:
|
||||
self._log(f"[Call {call_id}] API call attempt #{retry_count + 1} to {model_name_full}{' (retrying indefinitely)' if retry_indefinitely else ''}")
|
||||
|
||||
# Using client.models.generate_content like in FL_GeminiImageEditor
|
||||
response = client_instance.models.generate_content(
|
||||
model=model_name_full, # FL_GeminiImageEditor passes the full model string here
|
||||
contents=contents,
|
||||
config=gen_config_obj # FL_GeminiImageEditor uses 'config' for GenerateContentConfig
|
||||
)
|
||||
|
||||
# Validate response structure (adapted from FL_GeminiImageEditor)
|
||||
if not hasattr(response, 'candidates') or not response.candidates:
|
||||
self._log(f"[Call {call_id}] Empty response: No candidates found")
|
||||
if retry_indefinitely or retry_count < max_retries - 1:
|
||||
self._log(f"[Call {call_id}] Retrying in 2 seconds... (Attempt {retry_count + 2 if not retry_indefinitely else 'N/A'}/{max_retries if not retry_indefinitely else 'inf'})")
|
||||
time.sleep(2)
|
||||
return self._call_gemini_api(client_instance, model_name_full, contents, gen_config_obj, retry_indefinitely, retry_count + 1, max_retries, call_id)
|
||||
else:
|
||||
self._log(f"[Call {call_id}] Maximum retries ({max_retries}) reached. Returning empty response.")
|
||||
return None
|
||||
|
||||
if not hasattr(response.candidates[0], 'content') or response.candidates[0].content is None:
|
||||
self._log(f"[Call {call_id}] Invalid response: candidates[0].content is missing")
|
||||
if retry_indefinitely or retry_count < max_retries - 1:
|
||||
self._log(f"[Call {call_id}] Retrying in 2 seconds... (Attempt {retry_count + 2 if not retry_indefinitely else 'N/A'}/{max_retries if not retry_indefinitely else 'inf'})")
|
||||
time.sleep(2)
|
||||
return self._call_gemini_api(client_instance, model_name_full, contents, gen_config_obj, retry_indefinitely, retry_count + 1, max_retries, call_id)
|
||||
else:
|
||||
self._log(f"[Call {call_id}] Maximum retries ({max_retries}) reached. Returning empty response.")
|
||||
return None
|
||||
|
||||
if not hasattr(response.candidates[0].content, 'parts') or response.candidates[0].content.parts is None:
|
||||
self._log(f"[Call {call_id}] Invalid response: candidates[0].content.parts is missing")
|
||||
if retry_indefinitely or retry_count < max_retries - 1:
|
||||
self._log(f"[Call {call_id}] Retrying in 2 seconds... (Attempt {retry_count + 2 if not retry_indefinitely else 'N/A'}/{max_retries if not retry_indefinitely else 'inf'})")
|
||||
time.sleep(2)
|
||||
return self._call_gemini_api(client_instance, model_name_full, contents, gen_config_obj, retry_indefinitely, retry_count + 1, max_retries, call_id)
|
||||
else:
|
||||
self._log(f"[Call {call_id}] Maximum retries ({max_retries}) reached. Returning empty response.")
|
||||
return None
|
||||
|
||||
self._log(f"[Call {call_id}] Valid API response received.")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] API call error: {str(e)}")
|
||||
if retry_indefinitely or retry_count < max_retries - 1:
|
||||
wait_time = 2 * (retry_count + 1) # Progressive backoff
|
||||
self._log(f"[Call {call_id}] Retrying in {wait_time}s... (Attempt {retry_count + 2 if not retry_indefinitely else 'N/A'}/{max_retries if not retry_indefinitely else 'inf'})")
|
||||
time.sleep(wait_time)
|
||||
return self._call_gemini_api(client_instance, model_name_full, contents, gen_config_obj, retry_indefinitely, retry_count + 1, max_retries, call_id)
|
||||
else:
|
||||
self._log(f"[Call {call_id}] Max retries ({max_retries}) reached. Giving up.")
|
||||
return None
|
||||
|
||||
def _process_api_response(self, response, call_id="0", always_square=False):
|
||||
if response is None: # Simplified check from Editor
|
||||
self._log(f"[Call {call_id}] No valid response to process.")
|
||||
error_msg = "API Error: No content in response"
|
||||
return self._create_error_image(error_msg), error_msg
|
||||
|
||||
response_text_parts = [] # Changed from response_text to response_text_parts to match ADV logic initially
|
||||
image_tensor = None
|
||||
|
||||
if not hasattr(response, 'candidates') or not response.candidates: # Check from Editor
|
||||
self._log(f"[Call {call_id}] No candidates in API response")
|
||||
error_msg = "API returned an empty response"
|
||||
return self._create_error_image(error_msg), error_msg
|
||||
|
||||
# Iterate through response parts (similar to Editor, but adapted for ADV's single image focus per call)
|
||||
for part in response.candidates[0].content.parts:
|
||||
if hasattr(part, 'text') and part.text is not None:
|
||||
text_content = part.text
|
||||
response_text_parts.append(text_content)
|
||||
self._log(
|
||||
f"[Call {call_id}] API returned text: {text_content[:100]}..." if len(
|
||||
text_content) > 100 else text_content)
|
||||
|
||||
elif hasattr(part, 'inline_data') and part.inline_data is not None:
|
||||
self._log(f"[Call {call_id}] API returned image data")
|
||||
try:
|
||||
image_data = part.inline_data.data
|
||||
|
||||
if not image_data or len(image_data) < 100: # Check from Editor
|
||||
self._log(f"[Call {call_id}] Warning: Image data is empty or too small")
|
||||
continue
|
||||
|
||||
pil_image = None
|
||||
try:
|
||||
pil_image = Image.open(BytesIO(image_data))
|
||||
self._log(
|
||||
f"[Call {call_id}] Direct PIL open successful, size: {pil_image.width}x{pil_image.height}")
|
||||
except Exception as e1:
|
||||
self._log(f"[Call {call_id}] Direct PIL open failed: {str(e1)}")
|
||||
# Optional: Add temp file saving from Editor if direct open fails often
|
||||
try:
|
||||
temp_dir = tempfile.gettempdir()
|
||||
# Ensure temp_dir is writable, or fall back
|
||||
if not os.access(temp_dir, os.W_OK):
|
||||
temp_dir = "." # Current directory as fallback
|
||||
self._log(f"[Call {call_id}] Temp directory {tempfile.gettempdir()} not writable, using current directory.")
|
||||
|
||||
temp_file_path = os.path.join(temp_dir, f"gemini_image_adv_{call_id}_{int(time.time())}.png")
|
||||
with open(temp_file_path, "wb") as f:
|
||||
f.write(image_data)
|
||||
pil_image = Image.open(temp_file_path)
|
||||
self._log(f"[Call {call_id}] Opening via temp file {temp_file_path} successful")
|
||||
try:
|
||||
os.remove(temp_file_path) # Clean up temp file
|
||||
except Exception as e_remove:
|
||||
self._log(f"[Call {call_id}] Could not remove temp file {temp_file_path}: {e_remove}")
|
||||
except Exception as e2:
|
||||
self._log(f"[Call {call_id}] Opening via temp file failed: {str(e2)}")
|
||||
|
||||
|
||||
if pil_image is None:
|
||||
self._log(f"[Call {call_id}] Cannot open image, skipping")
|
||||
continue
|
||||
|
||||
if pil_image.mode != 'RGB':
|
||||
pil_image = pil_image.convert('RGB')
|
||||
self._log(f"[Call {call_id}] Image converted to RGB mode")
|
||||
|
||||
# Store original dimensions for logging
|
||||
width, height = pil_image.size
|
||||
self._log(f"[Call {call_id}] Original image size: {width}x{height}")
|
||||
|
||||
# Apply padding if always_square is enabled and image needs it
|
||||
if always_square and (width < self.min_size or height < self.min_size):
|
||||
self._log(
|
||||
f"[Call {call_id}] Image size {width}x{height} is smaller than minimum {self.min_size}x{self.min_size}, padding needed")
|
||||
pil_image = self._pad_image_to_minimum_size(pil_image)
|
||||
elif always_square:
|
||||
self._log(f"[Call {call_id}] Always square enabled but image already meets minimum size")
|
||||
else:
|
||||
self._log(f"[Call {call_id}] Always square disabled, keeping original size: {width}x{height}")
|
||||
|
||||
img_array = np.array(pil_image).astype(np.float32) / 255.0
|
||||
image_tensor = torch.from_numpy(img_array).unsqueeze(0) # Batch dimension
|
||||
self._log(f"[Call {call_id}] Image processed from API response. Shape: {image_tensor.shape}")
|
||||
break # Assuming one image per response for ADV node
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] Error processing image from API response: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
final_response_text = "\n".join(response_text_parts)
|
||||
if image_tensor is None:
|
||||
self._log(f"[Call {call_id}] No image found in API response parts.")
|
||||
error_msg = "API Error: No image data in response" # More specific than Editor's default
|
||||
image_tensor = self._create_error_image(error_msg) # Use the updated _create_error_image
|
||||
if not final_response_text: final_response_text = error_msg # Keep this logic
|
||||
|
||||
return image_tensor, final_response_text
|
||||
|
||||
async def _generate_single_image_async(self, api_key, model_name_full, prompt_text, input_pil_images: Optional[List[Image.Image]], temperature, max_retries, retry_indefinitely, seed_val, call_id, always_square=False, aspect_ratio="1:1", image_size="1K"):
|
||||
try:
|
||||
try:
|
||||
client_instance = genai.Client(api_key=api_key)
|
||||
except TypeError:
|
||||
genai.configure(api_key=api_key)
|
||||
client_instance = genai.Client()
|
||||
except AttributeError:
|
||||
self._log(f"[Call {call_id}] CRITICAL: genai.Client not found. Please check google-genai SDK installation and version.")
|
||||
error_msg = f"Call {call_id} Error: genai.Client not found."
|
||||
return self._create_error_image(error_msg), error_msg, call_id
|
||||
|
||||
actual_seed = seed_val if seed_val != 0 else random.randint(1, 0xffffff)
|
||||
self._log(f"[Call {call_id}] Using seed: {actual_seed} for prompt: '{prompt_text[:50]}...'")
|
||||
|
||||
gen_config_params = {
|
||||
"temperature": temperature,
|
||||
"response_modalities": ['Text', 'Image']
|
||||
}
|
||||
if actual_seed != 0:
|
||||
gen_config_params["seed"] = actual_seed
|
||||
|
||||
# Add image config if supported
|
||||
try:
|
||||
# Check if ImageConfig supports imageSize parameter (SDK >= 1.50)
|
||||
import inspect
|
||||
image_config_sig = inspect.signature(types.ImageConfig)
|
||||
supports_image_size = 'imageSize' in image_config_sig.parameters or 'image_size' in image_config_sig.parameters
|
||||
|
||||
image_config_params = {"aspectRatio": aspect_ratio}
|
||||
# Only add imageSize for gemini-3-pro model if SDK supports it
|
||||
if "gemini-3-pro" in model_name_full and supports_image_size:
|
||||
image_config_params["imageSize"] = image_size
|
||||
self._log(f"[Call {call_id}] Using imageSize: {image_size}")
|
||||
elif "gemini-3-pro" in model_name_full:
|
||||
self._log(f"[Call {call_id}] Warning: imageSize not supported in this SDK version. Upgrade google-genai to 1.50+ for 4K support")
|
||||
|
||||
gen_config_params["image_config"] = types.ImageConfig(**image_config_params)
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] ImageConfig not available: {e}, using basic config")
|
||||
|
||||
gen_config_obj = types.GenerateContentConfig(**gen_config_params)
|
||||
|
||||
if actual_seed != 0:
|
||||
current_seed_in_config = getattr(gen_config_obj, 'seed', None)
|
||||
if current_seed_in_config != actual_seed:
|
||||
self._log(f"[Call {call_id}] Warning: Seed {actual_seed} was specified. GenerateContentConfig has seed: {current_seed_in_config}. Ensure model supports seed via this config.")
|
||||
|
||||
contents = [prompt_text] # Start with the prompt
|
||||
if input_pil_images: # If the list of PIL images is provided and not empty
|
||||
for pil_img in input_pil_images:
|
||||
if pil_img: # Ensure the image itself is not None
|
||||
contents.append(pil_img)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
response = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self._call_gemini_api(client_instance, model_name_full, contents, gen_config_obj, retry_indefinitely, 0, max_retries, call_id)
|
||||
)
|
||||
|
||||
img_tensor, response_text = self._process_api_response(response, call_id, always_square)
|
||||
return img_tensor, response_text, call_id # Return call_id to map results
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"[Call {call_id}] Error in async generation: {str(e)}")
|
||||
error_msg = f"Call {call_id} Error: {str(e)}"
|
||||
return self._create_error_image(error_msg), error_msg, call_id
|
||||
|
||||
def generate_images_advanced(self, inputcount, api_key, model, aspect_ratio, image_size, always_square, temperature, max_retries, prompt_1, image_1=None, seed=0, retry_indefinitely=False, **kwargs):
|
||||
self.log_messages = []
|
||||
if not api_key:
|
||||
error_msg = "API key not provided."
|
||||
self._log(error_msg)
|
||||
error_img_instance = self._create_error_image(error_msg)
|
||||
# API key error should return 'inputcount' error images if we can determine it,
|
||||
# otherwise, a single error image is a reasonable fallback.
|
||||
# Since each slot is one API call now, inputcount is the number of expected results.
|
||||
return ([error_img_instance] * inputcount, error_msg)
|
||||
|
||||
pbar = ProgressBar(inputcount) # Initialize progress bar
|
||||
|
||||
# Setup async tasks for each input
|
||||
async def run_batch():
|
||||
tasks = []
|
||||
|
||||
for slot_idx in range(1, inputcount + 1):
|
||||
current_prompt = prompt_1 if slot_idx == 1 else kwargs.get(f"prompt_{slot_idx}", f"Default prompt for image {slot_idx}")
|
||||
|
||||
current_image_tensor_for_slot = None
|
||||
if slot_idx == 1:
|
||||
current_image_tensor_for_slot = image_1
|
||||
else:
|
||||
current_image_tensor_for_slot = kwargs.get(f"image_{slot_idx}")
|
||||
|
||||
pil_images_for_this_slot = self._process_tensor_to_pil_list(current_image_tensor_for_slot, f"InputSlot{slot_idx}")
|
||||
|
||||
current_task_seed = seed + (slot_idx - 1) if seed != 0 else 0
|
||||
task_call_id = str(slot_idx)
|
||||
|
||||
tasks.append(self._generate_single_image_async(
|
||||
api_key, model, current_prompt, pil_images_for_this_slot,
|
||||
temperature, max_retries, retry_indefinitely, current_task_seed, task_call_id, always_square,
|
||||
aspect_ratio, image_size
|
||||
))
|
||||
pbar.update_absolute(slot_idx) # Update progress bar after task is added
|
||||
|
||||
if not tasks:
|
||||
self._log("No tasks were created. This might indicate an issue with inputcount or logic.")
|
||||
return []
|
||||
|
||||
# Run all tasks concurrently
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
# Run the async batch processing using thread pool to avoid event loop conflicts
|
||||
def run_sync_batch():
|
||||
"""Run async batch in a new thread with its own event loop"""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return loop.run_until_complete(run_batch())
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
results_with_id = None # Initialize results
|
||||
try:
|
||||
# Use thread pool executor to run async code in separate thread
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(run_sync_batch)
|
||||
results_with_id = future.result(timeout=300) # 5 minute timeout
|
||||
except concurrent.futures.TimeoutError:
|
||||
self._log("Async processing timed out after 5 minutes")
|
||||
error_imgs = [self._create_error_image("Processing timeout")] * inputcount
|
||||
return (error_imgs, "Processing timed out after 5 minutes")
|
||||
except Exception as e:
|
||||
self._log(f"Error in async processing: {str(e)}")
|
||||
# Create batch of error images
|
||||
error_imgs = [self._create_error_image(f"Async processing error: {str(e)}")] * inputcount
|
||||
return (error_imgs, f"Async processing error: {str(e)}")
|
||||
|
||||
# Process results (ensure results is not None if an error occurred before assignment)
|
||||
if results_with_id is None:
|
||||
self._log("Async processing did not yield results, possibly due to an earlier error before gather.")
|
||||
error_imgs = [self._create_error_image("Async processing failed to produce results")] * inputcount
|
||||
return (error_imgs, "Async processing failed to produce results")
|
||||
|
||||
results_with_id.sort(key=lambda x: int(x[2]))
|
||||
|
||||
output_images = []
|
||||
output_texts = []
|
||||
|
||||
for img_tensor, response_text, call_id_res in results_with_id:
|
||||
output_images.append(img_tensor)
|
||||
output_texts.append(f"Response for Input {call_id_res}:\n{response_text}")
|
||||
|
||||
# Return as list of individual tensors instead of batched tensor to match FL_GeminiImageEditor behavior
|
||||
combined_responses = "\n\n".join(output_texts)
|
||||
|
||||
final_log_output = "Processing Logs:\n" + "\n".join(self.log_messages) + "\n\n" + combined_responses
|
||||
|
||||
return (output_images if output_images else [self._create_error_image("No images generated")], final_log_output)
|
||||
@@ -1,156 +0,0 @@
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from PIL import Image
|
||||
from google import genai
|
||||
|
||||
import comfy.model_management as mm
|
||||
from comfy_api.latest import Types
|
||||
|
||||
from ._language_models import GEMINI_LANGUAGE_MODELS, model_choices, validate_gemini_model
|
||||
|
||||
|
||||
class FL_GeminiTextAPI:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt": ("STRING", {"multiline": True}),
|
||||
"api_key": ("STRING", {"default": "", "multiline": False}),
|
||||
"model": (model_choices(GEMINI_LANGUAGE_MODELS), {"default": "gemini-3.7-flash"}),
|
||||
"temperature": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.05}),
|
||||
"max_output_tokens": ("INT", {"default": 8192, "min": 64, "max": 65536, "step": 64}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffff}),
|
||||
},
|
||||
"optional": {
|
||||
"system_instructions": ("STRING", {"multiline": True, "default": ""}),
|
||||
"top_p": ("FLOAT", {"default": 0.95, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"top_k": ("INT", {"default": 64, "min": 1, "max": 100, "step": 1}),
|
||||
"thinking_level": (["default", "low", "medium", "high"], {"default": "default"}),
|
||||
"custom_model": ("STRING", {"default": "", "placeholder": "Optional Gemini model ID override"}),
|
||||
"audio": ("AUDIO", {"tooltip": "Send the full audio to Gemini. Each batch item is a separate recording, in order."}),
|
||||
"image": ("IMAGE", {"tooltip": "Send every image in the batch, in order. Use VIDEO for frames with timing and sound."}),
|
||||
"video": ("VIDEO", {"tooltip": "Send a native ComfyUI video, including its audio and selected trim. Connect Load Video or Create Video."}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("response",)
|
||||
FUNCTION = "generate_text"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message):
|
||||
print(f"[FL_GeminiTextAPI] {time.strftime('%Y-%m-%d %H:%M:%S')}: {message}")
|
||||
self.log_messages.append(message)
|
||||
|
||||
@staticmethod
|
||||
def _media_files(directory, audio, image, video):
|
||||
files = []
|
||||
if audio is not None:
|
||||
waveform, rate = audio["waveform"], int(audio["sample_rate"])
|
||||
if waveform.ndim != 3 or waveform.shape[0] == 0 or waveform.shape[1] not in (1, 2) or waveform.shape[2] == 0 or rate <= 0:
|
||||
raise ValueError("Audio must contain nonempty mono/stereo recordings with a positive sample rate.")
|
||||
for index, clip in enumerate(waveform):
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
samples = clip.detach().cpu().float().T.numpy()
|
||||
if not np.isfinite(samples).all():
|
||||
raise ValueError("Audio contains non-finite samples.")
|
||||
path = directory / f"audio-{index + 1}.flac"
|
||||
sf.write(path, samples, rate, subtype="PCM_24")
|
||||
files.append(("audio", path, "audio/flac"))
|
||||
if image is not None:
|
||||
if image.ndim != 4 or image.shape[0] == 0 or image.shape[-1] not in (1, 3, 4):
|
||||
raise ValueError("Image must be a nonempty batch of grayscale, RGB, or RGBA images.")
|
||||
for index, frame in enumerate(image):
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
pixels = frame.detach().cpu().clamp(0, 1).mul(255).byte().numpy()
|
||||
if pixels.shape[-1] == 1:
|
||||
pixels = pixels[..., 0]
|
||||
path = directory / f"image-{index + 1}.png"
|
||||
Image.fromarray(pixels).save(path)
|
||||
files.append(("image", path, "image/png"))
|
||||
if video is not None:
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
path = directory / "video.mp4"
|
||||
video.save_to(str(path), format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264)
|
||||
files.append(("video", path, "video/mp4"))
|
||||
return files
|
||||
|
||||
@staticmethod
|
||||
def _upload(client, path, mime_type, uploaded):
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
file = client.files.upload(file=str(path), config={"mime_type": mime_type})
|
||||
uploaded.append(file.name)
|
||||
deadline = time.monotonic() + 300
|
||||
while file.state.name == "PROCESSING":
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("Gemini media processing timed out; try a shorter clip.")
|
||||
time.sleep(1)
|
||||
file = client.files.get(name=file.name)
|
||||
if file.state.name != "ACTIVE":
|
||||
raise ValueError("Gemini could not process the uploaded media.")
|
||||
return file.uri
|
||||
|
||||
def generate_text(self, prompt, api_key, model, temperature, max_output_tokens, seed,
|
||||
system_instructions="", top_p=0.95, top_k=64, thinking_level="default", custom_model="",
|
||||
audio=None, image=None, video=None):
|
||||
self.log_messages = []
|
||||
if not api_key:
|
||||
return ("Error: No Google API key provided.",)
|
||||
|
||||
try:
|
||||
model, capability = validate_gemini_model(model, custom_model)
|
||||
for kind, media in (("audio", audio), ("image", image), ("video", video)):
|
||||
if media is not None:
|
||||
validate_gemini_model(model, required_input=kind)
|
||||
config = {
|
||||
"temperature": temperature,
|
||||
"max_output_tokens": min(max_output_tokens, capability.max_output_tokens),
|
||||
"top_p": top_p,
|
||||
"top_k": top_k,
|
||||
"seed": seed or None,
|
||||
}
|
||||
if system_instructions.strip():
|
||||
config["system_instruction"] = system_instructions
|
||||
if thinking_level != "default" and thinking_level in capability.thinking_levels:
|
||||
config["thinking_config"] = {"thinking_level": thinking_level.upper()}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="fl-gemini-") as temporary, genai.Client(api_key=api_key) as client:
|
||||
uploaded = []
|
||||
try:
|
||||
files = self._media_files(Path(temporary), audio, image, video)
|
||||
inputs = [{"type": "text", "text": prompt}]
|
||||
for kind, path, mime_type in files:
|
||||
uri = self._upload(client, path, mime_type, uploaded)
|
||||
inputs.append({"type": kind, "uri": uri, "mime_type": mime_type})
|
||||
mm.throw_exception_if_processing_interrupted()
|
||||
self._log(f"Sending request to {model} with {len(files)} media file(s).")
|
||||
interaction = client.interactions.create(
|
||||
model=model, store=False, timeout=300,
|
||||
input=inputs if files else prompt,
|
||||
generation_config={key: value for key, value in config.items() if value is not None},
|
||||
)
|
||||
finally:
|
||||
for name in uploaded:
|
||||
try:
|
||||
client.files.delete(name=name)
|
||||
except Exception:
|
||||
self._log("Temporary Google file cleanup failed; remove the file in Google AI Studio.")
|
||||
text = getattr(interaction, "output_text", "")
|
||||
if not text:
|
||||
raise ValueError("Gemini returned no text output.")
|
||||
return (text.strip(),)
|
||||
except mm.InterruptProcessingException:
|
||||
raise
|
||||
except Exception as error:
|
||||
self._log(f"Error: {error}")
|
||||
traceback.print_exc()
|
||||
return (f"Error: {error}",)
|
||||
@@ -1,106 +0,0 @@
|
||||
import base64
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import torch
|
||||
from google import genai
|
||||
|
||||
from ._language_models import GEMINI_LANGUAGE_MODELS, model_choices, validate_gemini_model
|
||||
|
||||
|
||||
class FL_GeminiVideoCaptioner:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"default": "", "multiline": False}),
|
||||
"model": (model_choices(GEMINI_LANGUAGE_MODELS), {"default": "gemini-3.7-flash"}),
|
||||
"frames_per_second": ("FLOAT", {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}),
|
||||
"max_duration_minutes": ("FLOAT", {"default": 2.0, "min": 0.1, "max": 45.0, "step": 0.1}),
|
||||
"prompt": ("STRING", {"default": "Describe this video scene in detail. Include important actions, subjects, settings, and atmosphere.", "multiline": True}),
|
||||
"process_audio": (["false", "true"], {"default": "false"}),
|
||||
"temperature": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.1}),
|
||||
"max_output_tokens": ("INT", {"default": 8192, "min": 50, "max": 65536, "step": 64}),
|
||||
"top_p": ("FLOAT", {"default": 0.95, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"top_k": ("INT", {"default": 64, "min": 1, "max": 100, "step": 1}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffff}),
|
||||
"thinking_level": (["default", "low", "medium", "high"], {"default": "default"}),
|
||||
},
|
||||
"optional": {
|
||||
"video_path": ("STRING", {"default": ""}),
|
||||
"image": ("IMAGE", {}),
|
||||
"custom_model": ("STRING", {"default": "", "placeholder": "Optional Gemini model ID override"}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING", "IMAGE")
|
||||
RETURN_NAMES = ("caption", "sampled_frame")
|
||||
FUNCTION = "generate_video_caption"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
@staticmethod
|
||||
def _image_input(frame):
|
||||
success, encoded = cv2.imencode(".png", cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
|
||||
if not success:
|
||||
raise ValueError("Could not encode video frame.")
|
||||
return {"type": "image", "data": base64.b64encode(encoded.tobytes()).decode(), "mime_type": "image/png"}
|
||||
|
||||
@staticmethod
|
||||
def _sample_frame(video_path):
|
||||
capture = cv2.VideoCapture(video_path)
|
||||
frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
capture.set(cv2.CAP_PROP_POS_FRAMES, max(0, frame_count // 2))
|
||||
success, frame = capture.read()
|
||||
capture.release()
|
||||
if not success:
|
||||
raise ValueError("Could not read a frame from video.")
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
return frame, torch.from_numpy(frame.astype(np.float32) / 255.0).unsqueeze(0)
|
||||
|
||||
def generate_video_caption(self, api_key, model, frames_per_second, max_duration_minutes, prompt,
|
||||
process_audio, temperature, max_output_tokens, top_p, top_k, seed,
|
||||
thinking_level, video_path=None, image=None, custom_model=""):
|
||||
if not api_key:
|
||||
raise ValueError("Gemini API key is required")
|
||||
model, capability = validate_gemini_model(model, custom_model, "video" if video_path else "image")
|
||||
client = genai.Client(api_key=api_key)
|
||||
uploaded = None
|
||||
try:
|
||||
config = {"temperature": temperature, "max_output_tokens": min(max_output_tokens, capability.max_output_tokens), "top_p": top_p, "top_k": top_k, "seed": seed or None}
|
||||
if thinking_level != "default" and thinking_level in capability.thinking_levels:
|
||||
config["thinking_config"] = {"thinking_level": thinking_level.upper()}
|
||||
if video_path:
|
||||
if not os.path.isfile(video_path):
|
||||
raise ValueError("video_path does not exist")
|
||||
frame, sample = self._sample_frame(video_path)
|
||||
mime_type = mimetypes.guess_type(video_path)[0] or "video/mp4"
|
||||
uploaded = client.files.upload(file=video_path, config={"mime_type": mime_type})
|
||||
while getattr(uploaded, "state", None) and getattr(uploaded.state, "name", uploaded.state) == "PROCESSING":
|
||||
time.sleep(2)
|
||||
uploaded = client.files.get(name=uploaded.name)
|
||||
if getattr(getattr(uploaded, "state", None), "name", getattr(uploaded, "state", None)) == "FAILED":
|
||||
raise ValueError("Gemini could not process the video file.")
|
||||
inputs = [{"type": "video", "uri": uploaded.uri, "mime_type": uploaded.mime_type}, {"type": "text", "text": prompt}]
|
||||
elif image is not None:
|
||||
frames = [(frame.cpu().numpy().clip(0, 1) * 255).astype(np.uint8) for frame in image]
|
||||
if not frames:
|
||||
raise ValueError("image input is empty")
|
||||
sample = image[len(image) // 2].unsqueeze(0)
|
||||
stride = max(1, round(24 / frames_per_second))
|
||||
inputs = [{"type": "text", "text": prompt}] + [self._image_input(frame) for frame in frames[::stride]]
|
||||
else:
|
||||
raise ValueError("Provide video_path or image input.")
|
||||
interaction = client.interactions.create(model=model, input=inputs, generation_config={key: value for key, value in config.items() if value is not None})
|
||||
caption = getattr(interaction, "output_text", "")
|
||||
if not caption:
|
||||
raise ValueError("Gemini returned no caption.")
|
||||
return (caption.strip(), sample)
|
||||
finally:
|
||||
if uploaded is not None:
|
||||
try:
|
||||
client.files.delete(name=uploaded.name)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -1,363 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import logging # Keep for now, can be replaced by self._log
|
||||
import os
|
||||
import time
|
||||
import tempfile
|
||||
import traceback # Import traceback
|
||||
from typing import Dict, Optional, Union, Any
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import cv2 # For video frame extraction
|
||||
|
||||
# ComfyUI specific imports
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
# Attempt to load dotenv for local dev, but API key primarily from input
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class FL_Hedra_API: # Renamed class
|
||||
BASE_URL = "https://api.hedra.com/web-app/public"
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"default": os.getenv("HEDRA_API_KEY", ""), "multiline": False}),
|
||||
"image": ("IMAGE",),
|
||||
"audio_file_path": ("STRING", {"default": "assets/audio.mp3", "multiline": False}),
|
||||
"text_prompt": ("STRING", {"default": "character talking on a white background", "multiline": True}),
|
||||
"aspect_ratio": (["1:1", "16:9", "9:16"], {"default": "1:1"}),
|
||||
"resolution": (["720p", "540p"], {"default": "720p"}),
|
||||
},
|
||||
"optional": {
|
||||
"duration_seconds": ("FLOAT", {"default": 0.0, "min": 0.0, "step": 0.1, "precision": 1}), # 0.0 means API default
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffff}), # 0 means API default/random
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("frames", "api_log")
|
||||
FUNCTION = "execute_hedra_pipeline"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
DESCRIPTION = "Generates a video using the Hedra API from an image, audio, and prompt, then outputs its frames."
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message: str, level: str = "INFO"):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_Hedra_API] {timestamp} [{level}]: {message}" # Updated log prefix
|
||||
print(formatted_message)
|
||||
self.log_messages.append(formatted_message)
|
||||
|
||||
class _HedraSession(requests.Session):
|
||||
def __init__(self, api_key: str, base_url: str):
|
||||
super().__init__()
|
||||
self.base_url = base_url
|
||||
self.headers["x-api-key"] = api_key
|
||||
|
||||
def prepare_request(self, request: requests.Request) -> requests.PreparedRequest:
|
||||
if not request.url.startswith(("http://", "https://")):
|
||||
request.url = f"{self.base_url.rstrip('/')}/{request.url.lstrip('/')}"
|
||||
return super().prepare_request(request)
|
||||
|
||||
def _convert_tensor_to_temp_image(self, image_tensor: torch.Tensor) -> Optional[str]:
|
||||
if image_tensor is None:
|
||||
return None
|
||||
try:
|
||||
img_np = image_tensor[0].cpu().numpy() # Assuming batch size 1 or taking the first image
|
||||
img_np = (img_np * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(img_np)
|
||||
|
||||
fd, temp_image_path = tempfile.mkstemp(suffix=".png", dir=tempfile.gettempdir())
|
||||
os.close(fd) # Close the file descriptor as PIL will open/close it
|
||||
pil_image.save(temp_image_path, format='PNG')
|
||||
self._log(f"Saved input tensor to temporary image: {temp_image_path}")
|
||||
return temp_image_path
|
||||
except Exception as e:
|
||||
self._log(f"Error converting tensor to temp image: {e}", level="ERROR")
|
||||
return None
|
||||
|
||||
def _upload_asset(self, session: _HedraSession, file_path: str, asset_type: str, asset_name: Optional[str] = None) -> Optional[str]:
|
||||
if not os.path.exists(file_path):
|
||||
self._log(f"Asset file not found: {file_path}", level="ERROR")
|
||||
return None
|
||||
|
||||
asset_name = asset_name or os.path.basename(file_path)
|
||||
self._log(f"Uploading {asset_type}: {asset_name} from {file_path}")
|
||||
|
||||
try:
|
||||
asset_response = session.post("/assets", json={"name": asset_name, "type": asset_type})
|
||||
asset_response.raise_for_status()
|
||||
asset_id = asset_response.json()["id"]
|
||||
|
||||
with open(file_path, "rb") as f:
|
||||
upload_response = session.post(f"/assets/{asset_id}/upload", files={"file": f})
|
||||
upload_response.raise_for_status()
|
||||
|
||||
self._log(f"Successfully uploaded {asset_type} with ID: {asset_id}")
|
||||
return asset_id
|
||||
except requests.exceptions.RequestException as e:
|
||||
self._log(f"Error during {asset_type} asset upload: {e}. Response: {e.response.text if e.response else 'N/A'}", level="ERROR")
|
||||
except Exception as e:
|
||||
self._log(f"Unexpected error during {asset_type} asset upload: {e}", level="ERROR")
|
||||
return None
|
||||
|
||||
def _get_available_models(self, session: _HedraSession) -> list:
|
||||
self._log("Fetching available models...")
|
||||
try:
|
||||
response = session.get("/models")
|
||||
response.raise_for_status()
|
||||
models = response.json()
|
||||
self._log(f"Found {len(models)} available models.")
|
||||
return models
|
||||
except requests.exceptions.RequestException as e:
|
||||
self._log(f"Error fetching models: {e}", level="ERROR")
|
||||
return []
|
||||
|
||||
def _generate_video_request(self, session: _HedraSession, model_id: str, image_id: str, audio_id: str,
|
||||
text_prompt: str, resolution: str, aspect_ratio: str,
|
||||
duration: Optional[float] = None, seed: Optional[int] = None) -> Optional[Dict[str, Any]]:
|
||||
self._log("Submitting video generation request...")
|
||||
payload = {
|
||||
"type": "video",
|
||||
"ai_model_id": model_id,
|
||||
"start_keyframe_id": image_id,
|
||||
"audio_id": audio_id,
|
||||
"generated_video_inputs": {
|
||||
"text_prompt": text_prompt,
|
||||
"resolution": resolution,
|
||||
"aspect_ratio": aspect_ratio,
|
||||
},
|
||||
}
|
||||
if duration and duration > 0.0: # API expects duration_ms > 0
|
||||
payload["generated_video_inputs"]["duration_ms"] = int(duration * 1000)
|
||||
if seed and seed != 0: # API might treat 0 as random/unset
|
||||
payload["generated_video_inputs"]["seed"] = seed
|
||||
|
||||
try:
|
||||
response = session.post("/generations", json=payload)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
self._log(f"Generation request submitted with ID: {result['id']}")
|
||||
return result
|
||||
except requests.exceptions.RequestException as e:
|
||||
self._log(f"Error submitting generation request: {e}. Response: {e.response.text if e.response else 'N/A'}", level="ERROR")
|
||||
return None
|
||||
|
||||
def _poll_generation_status(self, session: _HedraSession, generation_id: str, pbar: ProgressBar) -> Optional[Dict[str, Any]]:
|
||||
self._log(f"Polling generation status for ID: {generation_id}")
|
||||
poll_interval = 5 # seconds
|
||||
total_polls = 0 # For progress bar update
|
||||
max_polls_for_pbar = 60 # Arbitrary limit for pbar updates (5 mins)
|
||||
|
||||
while True:
|
||||
try:
|
||||
status_response = session.get(f"/generations/{generation_id}/status")
|
||||
status_response.raise_for_status()
|
||||
status_data = status_response.json()
|
||||
status = status_data.get("status", "unknown")
|
||||
progress = status_data.get("progress") # Hedra might provide progress %
|
||||
|
||||
log_msg = f"Current status: {status}"
|
||||
if progress is not None:
|
||||
log_msg += f" (Progress: {progress}%)"
|
||||
if isinstance(progress, (int, float)) and 0 <= progress <= 100:
|
||||
# Use the total value the pbar was initialized with for scaling
|
||||
# Assuming pbar here refers to the polling_pbar passed to this method
|
||||
# which was initialized with polling_pbar_total
|
||||
pbar_total_for_scaling = pbar.total # ProgressBar stores its total in pbar.total
|
||||
pbar.update_absolute(int(progress * (pbar_total_for_scaling / 100.0)))
|
||||
else:
|
||||
# Fallback pbar update if no explicit progress
|
||||
pbar.update_absolute(min(total_polls, max_polls_for_pbar))
|
||||
|
||||
self._log(log_msg)
|
||||
|
||||
if status in ["complete", "error"]:
|
||||
if status == "complete": pbar.update_absolute(pbar.total) # Ensure 100% on complete
|
||||
return status_data
|
||||
|
||||
time.sleep(poll_interval)
|
||||
total_polls +=1
|
||||
except requests.exceptions.RequestException as e:
|
||||
self._log(f"Error polling status: {e}", level="ERROR")
|
||||
return {"status": "error", "error_message": f"Polling failed: {e}"}
|
||||
except Exception as e:
|
||||
self._log(f"Unexpected error during polling: {e}", level="ERROR")
|
||||
return {"status": "error", "error_message": f"Unexpected polling error: {e}"}
|
||||
|
||||
|
||||
def _download_video_file(self, download_url: str, output_filename: str) -> Optional[str]:
|
||||
self._log(f"Downloading video from {download_url} to {output_filename}")
|
||||
try:
|
||||
with requests.get(download_url, stream=True, timeout=300) as r: # 5 min timeout for download
|
||||
r.raise_for_status()
|
||||
with open(output_filename, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
self._log(f"Successfully downloaded video to {output_filename}")
|
||||
return output_filename
|
||||
except requests.exceptions.RequestException as e:
|
||||
self._log(f"Failed to download video: {e}", level="ERROR")
|
||||
except IOError as e:
|
||||
self._log(f"Failed to save video file: {e}", level="ERROR")
|
||||
return None
|
||||
|
||||
def _extract_frames_from_video(self, video_path: str) -> Optional[torch.Tensor]:
|
||||
if not os.path.exists(video_path):
|
||||
self._log(f"Video file not found for frame extraction: {video_path}", level="ERROR")
|
||||
return None
|
||||
|
||||
self._log(f"Extracting frames from {video_path}")
|
||||
try:
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
frames = []
|
||||
pbar_frames = ProgressBar(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)))
|
||||
frame_count = 0
|
||||
while cap.isOpened():
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
pil_image = Image.fromarray(frame_rgb)
|
||||
img_array = np.array(pil_image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0) # Add batch dim
|
||||
frames.append(img_tensor)
|
||||
frame_count +=1
|
||||
pbar_frames.update_absolute(frame_count)
|
||||
cap.release()
|
||||
|
||||
if not frames:
|
||||
self._log("No frames extracted from video.", level="WARNING")
|
||||
return None
|
||||
|
||||
self._log(f"Extracted {len(frames)} frames.")
|
||||
return torch.cat(frames, dim=0)
|
||||
except Exception as e:
|
||||
self._log(f"Error extracting frames: {e}", level="ERROR")
|
||||
return None
|
||||
|
||||
def execute_hedra_pipeline(self, api_key: str, image: torch.Tensor, audio_file_path: str,
|
||||
text_prompt: str, aspect_ratio: str, resolution: str,
|
||||
duration_seconds: float = 0.0, seed: int = 0):
|
||||
self.log_messages = [] # Reset logs for this run
|
||||
|
||||
if not api_key:
|
||||
self._log("API key not provided.", level="ERROR")
|
||||
return (None, "API key not provided.")
|
||||
|
||||
session = self._HedraSession(api_key=api_key, base_url=self.BASE_URL)
|
||||
|
||||
temp_image_path = None
|
||||
temp_video_path = None
|
||||
final_frames = None
|
||||
|
||||
# Progress bar for the whole operation (approximated steps)
|
||||
# 1: upload image, 2: upload audio, 3: get models, 4: submit generation, 5: poll (scaled), 6: download, 7: extract frames
|
||||
# Polling itself will have its own progress if API provides it, or timed progress.
|
||||
# Let's use a simple step-based pbar for overall, and another for polling/extraction.
|
||||
overall_pbar = ProgressBar(7)
|
||||
|
||||
try:
|
||||
# 1. Upload Image
|
||||
temp_image_path = self._convert_tensor_to_temp_image(image)
|
||||
if not temp_image_path:
|
||||
raise RuntimeError("Failed to convert input image tensor.")
|
||||
image_id = self._upload_asset(session, temp_image_path, "image", "input_image.png")
|
||||
if not image_id:
|
||||
raise RuntimeError("Failed to upload image asset.")
|
||||
overall_pbar.update_absolute(1)
|
||||
|
||||
# 2. Upload Audio
|
||||
audio_id = self._upload_asset(session, audio_file_path, "audio")
|
||||
if not audio_id:
|
||||
raise RuntimeError(f"Failed to upload audio asset from {audio_file_path}.")
|
||||
overall_pbar.update_absolute(2)
|
||||
|
||||
# 3. Get Model ID
|
||||
models = self._get_available_models(session)
|
||||
if not models:
|
||||
raise RuntimeError("No models available from Hedra API.")
|
||||
model_id = models[0]["id"] # Use the first available model as per script
|
||||
self._log(f"Using model ID: {model_id}")
|
||||
overall_pbar.update_absolute(3)
|
||||
|
||||
# 4. Generate Video Request
|
||||
# For polling pbar, let's assume 100 steps for the polling phase itself
|
||||
polling_pbar_total = 100
|
||||
polling_pbar = ProgressBar(polling_pbar_total)
|
||||
|
||||
generation_request = self._generate_video_request(
|
||||
session, model_id, image_id, audio_id, text_prompt,
|
||||
resolution, aspect_ratio, duration_seconds if duration_seconds > 0 else None,
|
||||
seed if seed != 0 else None
|
||||
)
|
||||
if not generation_request or "id" not in generation_request:
|
||||
raise RuntimeError("Failed to submit video generation request.")
|
||||
generation_id = generation_request["id"]
|
||||
overall_pbar.update_absolute(4)
|
||||
|
||||
# 5. Poll Generation Status
|
||||
self._log("Starting to poll generation status...")
|
||||
final_status = self._poll_generation_status(session, generation_id, polling_pbar)
|
||||
if not final_status or final_status.get("status") != "complete":
|
||||
error_msg = final_status.get("error_message", "Video generation did not complete successfully or status unknown.")
|
||||
raise RuntimeError(f"Generation failed or status incomplete: {error_msg}")
|
||||
|
||||
download_url = final_status.get("url")
|
||||
if not download_url:
|
||||
raise RuntimeError("Generation complete but no download URL provided.")
|
||||
overall_pbar.update_absolute(5)
|
||||
|
||||
# 6. Download Video
|
||||
temp_video_dir = tempfile.gettempdir()
|
||||
temp_video_path = os.path.join(temp_video_dir, f"hedra_video_{generation_id}.mp4")
|
||||
downloaded_path = self._download_video_file(download_url, temp_video_path)
|
||||
if not downloaded_path:
|
||||
raise RuntimeError("Failed to download generated video.")
|
||||
overall_pbar.update_absolute(6)
|
||||
|
||||
# 7. Extract Frames
|
||||
final_frames = self._extract_frames_from_video(downloaded_path)
|
||||
if final_frames is None:
|
||||
raise RuntimeError("Failed to extract frames from downloaded video.")
|
||||
overall_pbar.update_absolute(7)
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Pipeline error: {e}", level="CRITICAL")
|
||||
traceback.print_exc() # For more detailed debug in console
|
||||
# Create a single black error frame if pipeline fails
|
||||
error_img_pil = Image.new('RGB', (256, 256), color='black') # Small error frame
|
||||
draw = ImageDraw.Draw(error_img_pil)
|
||||
try: font = ImageFont.load_default()
|
||||
except: font = None
|
||||
draw.text((10,10), f"Error: {str(e)[:100]}", fill="red", font=font)
|
||||
error_img_np = np.array(error_img_pil).astype(np.float32) / 255.0
|
||||
final_frames = torch.from_numpy(error_img_np).unsqueeze(0)
|
||||
finally:
|
||||
if temp_image_path and os.path.exists(temp_image_path):
|
||||
try: os.remove(temp_image_path)
|
||||
except Exception as e_rem: self._log(f"Could not remove temp image {temp_image_path}: {e_rem}", "WARNING")
|
||||
if temp_video_path and os.path.exists(temp_video_path):
|
||||
try: os.remove(temp_video_path)
|
||||
except Exception as e_rem: self._log(f"Could not remove temp video {temp_video_path}: {e_rem}", "WARNING")
|
||||
|
||||
return (final_frames, "\n".join(self.log_messages))
|
||||
|
||||
# Mappings for ComfyUI
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"FL_Hedra_API": FL_Hedra_API # Renamed mapping key
|
||||
}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"FL_Hedra_API": "FL Hedra API" # Renamed display name
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,329 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import httpx
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from typing import Literal, Union, List, Dict, Any
|
||||
|
||||
from runwayml import RunwayML
|
||||
from runwayml._types import NotGiven, NOT_GIVEN
|
||||
from runwayml.types import TaskRetrieveResponse
|
||||
from runwayml._exceptions import APIError, APIStatusError, APIConnectionError, RateLimitError
|
||||
from comfy.utils import ProgressBar
|
||||
|
||||
|
||||
class FL_RunwayAct2:
|
||||
DEFAULT_RUNWAY_API_VERSION = "2024-11-06"
|
||||
MODEL_NAME: Literal["act_two"] = "act_two"
|
||||
RATIO_OPTIONS = ['1280:720', '720:1280', '960:960', '1104:832', '832:1104', '1584:672']
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False, "default": os.environ.get('RUNWAYML_API_SECRET', '')}),
|
||||
"character_type": (["video", "image"], {"default": "video"}),
|
||||
"ratio": (cls.RATIO_OPTIONS, {"default": "1280:720"}),
|
||||
"runway_api_version": ("STRING", {"multiline": False, "default": cls.DEFAULT_RUNWAY_API_VERSION}),
|
||||
},
|
||||
"optional": {
|
||||
"character_video": ("IMAGE",), # Will be converted to video format
|
||||
"character_image": ("IMAGE",),
|
||||
"reference_video": ("IMAGE",), # Will be converted to video format
|
||||
"body_control": ("BOOLEAN", {"default": True}),
|
||||
"expression_intensity": ("INT", {"default": 3, "min": 1, "max": 5}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 4294967295}),
|
||||
"public_figure_threshold": (["auto", "low", "NOT_GIVEN"], {"default": "NOT_GIVEN"}),
|
||||
"polling_interval": ("INT", {"default": 5, "min": 1, "max": 60}),
|
||||
"max_polling_attempts": ("INT", {"default": 60, "min": 1, "max": 120}),
|
||||
"timeout_sdk": ("FLOAT", {"default": 60.0, "min": 5.0, "max": 300.0, "step": 1.0}),
|
||||
"timeout_download": ("FLOAT", {"default": 120.0, "min": 5.0, "max": 600.0, "step": 1.0}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("video_frames", "status_text")
|
||||
FUNCTION = "generate_character_performance"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message: str):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
full_message = f"[FL_RunwayAct2] {timestamp}: {message}"
|
||||
print(full_message)
|
||||
self.log_messages.append(full_message)
|
||||
|
||||
def _create_error_image(self, error_message="API Failed", width=1280, height=720) -> torch.Tensor:
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
try:
|
||||
font = ImageFont.truetype("arial.ttf", 24)
|
||||
except IOError:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
text_bbox = draw.textbbox((0,0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log(f"Created error image: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _process_tensor_to_pil(self, tensor: torch.Tensor, name="Image") -> Union[Image.Image, None]:
|
||||
if tensor is None:
|
||||
return None
|
||||
|
||||
# Handle single image: [1, height, width, 3]
|
||||
if len(tensor.shape) == 4 and tensor.shape[0] == 1:
|
||||
image_np = tensor[0].cpu().numpy()
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
return Image.fromarray(image_np)
|
||||
|
||||
# Handle video batch: [frames, height, width, 3] - use first frame
|
||||
elif len(tensor.shape) == 4 and tensor.shape[0] > 1:
|
||||
self._log(f"{name} appears to be a video with {tensor.shape[0]} frames. Using first frame.")
|
||||
image_np = tensor[0].cpu().numpy()
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
return Image.fromarray(image_np)
|
||||
|
||||
# Handle single frame without batch dimension: [height, width, 3]
|
||||
elif len(tensor.shape) == 3:
|
||||
image_np = tensor.cpu().numpy()
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
return Image.fromarray(image_np)
|
||||
|
||||
self._log(f"Error: {name} tensor format not supported: {tensor.shape}")
|
||||
return None
|
||||
|
||||
def _pil_to_data_uri(self, pil_image: Image.Image, image_format="PNG") -> str:
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format=image_format)
|
||||
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
return f"data:image/{image_format.lower()};base64,{img_str}"
|
||||
|
||||
def _create_video_data_uri_from_tensor(self, tensor: torch.Tensor, name="Video") -> str:
|
||||
"""Convert video tensor to data URI - for now using first frame as image"""
|
||||
# Note: This is a simplified implementation. For full video support,
|
||||
# you would need to encode the entire tensor as a video file (MP4, etc.)
|
||||
# and create a proper video data URI. For now, we'll use the first frame.
|
||||
|
||||
if len(tensor.shape) == 4 and tensor.shape[0] > 1:
|
||||
# Video tensor: [frames, height, width, 3]
|
||||
self._log(f"{name} has {tensor.shape[0]} frames. Using first frame for API compatibility.")
|
||||
first_frame = tensor[0].cpu().numpy()
|
||||
first_frame = (first_frame * 255).astype(np.uint8)
|
||||
pil_image = Image.fromarray(first_frame)
|
||||
return self._pil_to_data_uri(pil_image, "PNG")
|
||||
else:
|
||||
# Single image, convert normally
|
||||
pil_image = self._process_tensor_to_pil(tensor, name)
|
||||
if pil_image:
|
||||
return self._pil_to_data_uri(pil_image, "PNG")
|
||||
else:
|
||||
raise ValueError(f"Could not process {name} tensor")
|
||||
|
||||
def _download_and_process_video(self, video_url: str, timeout: float) -> torch.Tensor:
|
||||
"""Download video and convert to image tensor (first frame for now)"""
|
||||
try:
|
||||
with httpx.Client() as download_client:
|
||||
response = download_client.get(video_url, follow_redirects=True, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
|
||||
# For now, we'll treat the downloaded content as an image
|
||||
# In a full implementation, you'd use video processing libraries
|
||||
pil_image = Image.open(io.BytesIO(response.content)).convert("RGB")
|
||||
img_array = np.array(pil_image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log("Video downloaded and processed successfully.")
|
||||
return img_tensor
|
||||
except Exception as e:
|
||||
self._log(f"Error downloading video: {str(e)}")
|
||||
raise
|
||||
|
||||
def generate_character_performance(
|
||||
self, api_key: str, character_type: str, ratio: str, runway_api_version: str,
|
||||
character_video: torch.Tensor = None, character_image: torch.Tensor = None,
|
||||
reference_video: torch.Tensor = None, body_control: bool = True,
|
||||
expression_intensity: int = 3, seed: int = 0, public_figure_threshold: str = "NOT_GIVEN",
|
||||
polling_interval: int = 5, max_polling_attempts: int = 60,
|
||||
timeout_sdk: float = 60.0, timeout_download: float = 120.0
|
||||
):
|
||||
self.log_messages = []
|
||||
|
||||
if not api_key or not api_key.strip():
|
||||
self._log("Error: RunwayML API key not provided.")
|
||||
return self._create_error_image("API key missing"), "\n".join(self.log_messages)
|
||||
|
||||
# Validate inputs based on character type
|
||||
if character_type == "video" and character_video is None:
|
||||
self._log("Error: Character video is required when character_type is 'video'.")
|
||||
return self._create_error_image("Character video missing"), "\n".join(self.log_messages)
|
||||
|
||||
if character_type == "image" and character_image is None:
|
||||
self._log("Error: Character image is required when character_type is 'image'.")
|
||||
return self._create_error_image("Character image missing"), "\n".join(self.log_messages)
|
||||
|
||||
if reference_video is None:
|
||||
self._log("Error: Reference video is required.")
|
||||
return self._create_error_image("Reference video missing"), "\n".join(self.log_messages)
|
||||
|
||||
current_runway_api_version = runway_api_version if runway_api_version and runway_api_version.strip() else self.DEFAULT_RUNWAY_API_VERSION
|
||||
|
||||
try:
|
||||
client = RunwayML(api_key=api_key.strip(), runway_version=current_runway_api_version, timeout=timeout_sdk)
|
||||
self._log(f"RunwayML client initialized. API Version: {client.runway_version}, Timeout: {timeout_sdk}s")
|
||||
except Exception as e:
|
||||
self._log(f"Failed to initialize RunwayML client: {str(e)}")
|
||||
return self._create_error_image("SDK Client Init Failed"), "\n".join(self.log_messages)
|
||||
|
||||
active_task_id = None
|
||||
|
||||
try:
|
||||
# Process character input
|
||||
character_payload: Dict[str, Any] = {"type": character_type}
|
||||
|
||||
if character_type == "video" and character_video is not None:
|
||||
try:
|
||||
character_payload["uri"] = self._create_video_data_uri_from_tensor(character_video, "Character Video")
|
||||
except Exception as e:
|
||||
self._log(f"Error processing character video: {str(e)}")
|
||||
return self._create_error_image("Failed to process character video"), "\n".join(self.log_messages)
|
||||
|
||||
elif character_type == "image" and character_image is not None:
|
||||
pil_img = self._process_tensor_to_pil(character_image, "Character Image")
|
||||
if pil_img:
|
||||
character_payload["uri"] = self._pil_to_data_uri(pil_img)
|
||||
else:
|
||||
self._log("Error processing character image")
|
||||
return self._create_error_image("Failed to process character image"), "\n".join(self.log_messages)
|
||||
|
||||
# Process reference video
|
||||
try:
|
||||
reference_payload = {
|
||||
"type": "video",
|
||||
"uri": self._create_video_data_uri_from_tensor(reference_video, "Reference Video")
|
||||
}
|
||||
except Exception as e:
|
||||
self._log(f"Error processing reference video: {str(e)}")
|
||||
return self._create_error_image("Failed to process reference video"), "\n".join(self.log_messages)
|
||||
|
||||
# Build create arguments
|
||||
create_args: Dict[str, Any] = {
|
||||
"model": self.MODEL_NAME,
|
||||
"character": character_payload,
|
||||
"reference": reference_payload,
|
||||
"ratio": ratio,
|
||||
"bodyControl": body_control,
|
||||
"expressionIntensity": expression_intensity,
|
||||
}
|
||||
|
||||
current_seed = NOT_GIVEN if seed == 0 else seed
|
||||
create_args["seed"] = current_seed
|
||||
|
||||
if public_figure_threshold != "NOT_GIVEN":
|
||||
create_args["contentModeration"] = {"publicFigureThreshold": public_figure_threshold}
|
||||
|
||||
self._log(f"Requesting character performance generation with model '{self.MODEL_NAME}'")
|
||||
self._log(f"Character type: {character_type}, Ratio: {ratio}, Body control: {body_control}")
|
||||
self._log(f"Expression intensity: {expression_intensity}")
|
||||
|
||||
# Submit the task using the working SDK method
|
||||
try:
|
||||
self._log("Submitting character performance request...")
|
||||
initial_response = client.character_performance.create(**create_args)
|
||||
active_task_id = initial_response.id
|
||||
self._log(f"Task submitted successfully! Task ID: {active_task_id}")
|
||||
except AttributeError as e:
|
||||
self._log(f"SDK method not found: {str(e)}")
|
||||
self._log("The character_performance method is not available in this SDK version.")
|
||||
return self._create_error_image("SDK Method Not Available"), "\n".join(self.log_messages)
|
||||
except Exception as e:
|
||||
self._log(f"Error submitting task: {str(e)}")
|
||||
return self._create_error_image("Task Submission Failed"), "\n".join(self.log_messages)
|
||||
|
||||
# Poll for completion
|
||||
self._log(f"Polling task status for Task ID: {active_task_id} (every {polling_interval}s)...")
|
||||
task_details: Union[TaskRetrieveResponse, None] = None
|
||||
pbar = ProgressBar(max_polling_attempts)
|
||||
|
||||
for attempt in range(max_polling_attempts):
|
||||
task_details = client.tasks.retrieve(id=active_task_id)
|
||||
progress_percent = 0.0
|
||||
if task_details.status == "RUNNING" and task_details.progress is not None:
|
||||
progress_percent = task_details.progress * 100
|
||||
|
||||
status_log = f"Attempt {attempt + 1}/{max_polling_attempts} - Status: {task_details.status}"
|
||||
if progress_percent > 0:
|
||||
status_log += f", Progress: {progress_percent:.2f}%"
|
||||
self._log(status_log)
|
||||
|
||||
if task_details.status == "SUCCEEDED":
|
||||
self._log("Task SUCCEEDED!")
|
||||
if task_details.output and len(task_details.output) > 0 and isinstance(task_details.output[0], str):
|
||||
video_url = task_details.output[0]
|
||||
self._log(f"Video URL: {video_url}")
|
||||
self._log(f"Downloading video...")
|
||||
try:
|
||||
video_tensor = self._download_and_process_video(video_url, timeout_download)
|
||||
return video_tensor, "\n".join(self.log_messages)
|
||||
except httpx.HTTPStatusError as e_http:
|
||||
self._log(f"Error downloading video: HTTP {e_http.response.status_code} - {e_http.response.text}")
|
||||
return self._create_error_image(f"Download Failed: {e_http.response.status_code}"), "\n".join(self.log_messages)
|
||||
except Exception as e_dl:
|
||||
self._log(f"An error occurred during video download: {str(e_dl)}")
|
||||
return self._create_error_image("Download Error"), "\n".join(self.log_messages)
|
||||
else:
|
||||
self._log("Task succeeded but no valid output URL found.")
|
||||
return self._create_error_image("No Output URL"), "\n".join(self.log_messages)
|
||||
|
||||
elif task_details.status in ["FAILED", "CANCELLED"]:
|
||||
self._log(f"Task {task_details.status}.")
|
||||
failure_reason = task_details.failure if hasattr(task_details, 'failure') and task_details.failure else 'N/A'
|
||||
failure_code = task_details.failure_code if hasattr(task_details, 'failure_code') and task_details.failure_code else 'N/A'
|
||||
self._log(f" Reason: {failure_reason}")
|
||||
self._log(f" Failure code: {failure_code}")
|
||||
return self._create_error_image(f"Task {task_details.status}: {failure_reason[:60]}"), "\n".join(self.log_messages)
|
||||
|
||||
time.sleep(polling_interval)
|
||||
|
||||
# If loop finishes without returning
|
||||
if task_details:
|
||||
self._log(f"Max poll attempts ({max_polling_attempts}) reached. Last known task status: {task_details.status}.")
|
||||
else:
|
||||
self._log(f"Max poll attempts ({max_polling_attempts}) reached. Could not retrieve task details after submission.")
|
||||
return self._create_error_image("Polling Timeout"), "\n".join(self.log_messages)
|
||||
|
||||
except APIConnectionError as e:
|
||||
self._log(f"RunwayML API Connection Error: {str(e)}")
|
||||
return self._create_error_image("API Connection Error"), "\n".join(self.log_messages)
|
||||
except RateLimitError as e:
|
||||
self._log(f"RunwayML Rate Limit Error: {str(e)}")
|
||||
return self._create_error_image("Rate Limit Error"), "\n".join(self.log_messages)
|
||||
except APIStatusError as e:
|
||||
self._log(f"RunwayML API Status Error: {str(e)}. Status Code: {e.status_code if hasattr(e, 'status_code') else 'N/A'}. Response: {e.response.text if hasattr(e, 'response') and hasattr(e.response, 'text') else 'N/A'}")
|
||||
return self._create_error_image(f"API Status Error: {e.status_code if hasattr(e, 'status_code') else ''}"), "\n".join(self.log_messages)
|
||||
except APIError as e:
|
||||
self._log(f"RunwayML API Error: {str(e)}")
|
||||
return self._create_error_image("Runway API Error"), "\n".join(self.log_messages)
|
||||
except Exception as e:
|
||||
self._log(f"An unexpected error occurred: {str(e)}")
|
||||
if active_task_id:
|
||||
self._log(f"This error occurred for Task ID: {active_task_id}")
|
||||
return self._create_error_image("Unexpected Node Error"), "\n".join(self.log_messages)
|
||||
finally:
|
||||
if 'client' in locals() and client is not None and hasattr(client, 'close'):
|
||||
try:
|
||||
client.close()
|
||||
self._log("RunwayML client closed.")
|
||||
except Exception as e_close:
|
||||
self._log(f"Error closing RunwayML client: {str(e_close)}")
|
||||
@@ -1,275 +0,0 @@
|
||||
import os
|
||||
import time
|
||||
import httpx # For downloading the image
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from typing import Literal, Union, List, Dict, Any
|
||||
|
||||
from runwayml import RunwayML
|
||||
from runwayml._types import NotGiven, NOT_GIVEN # Import NOT_GIVEN sentinel
|
||||
from runwayml.types import TaskRetrieveResponse # For type hinting polled task
|
||||
from runwayml._exceptions import APIError, APIStatusError, APIConnectionError, RateLimitError
|
||||
from comfy.utils import ProgressBar # Import ProgressBar
|
||||
|
||||
|
||||
class FL_RunwayImageAPI:
|
||||
# Default API version used by the SDK if not specified in constructor
|
||||
# From your script, it seems the SDK handles this well if client is initialized with runway_version
|
||||
DEFAULT_RUNWAY_API_VERSION = "2024-11-06"
|
||||
MODEL_NAME: Literal["gen4_image"] = "gen4_image"
|
||||
IMAGE_RATIO_OPTIONS = ['1920:1080', '1080:1920', '1024:1024', '1360:768', '1080:1080', '1168:880', '1440:1080', '1080:1440', '1808:768', '2112:912']
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"api_key": ("STRING", {"multiline": False, "default": os.environ.get('RUNWAYML_API_SECRET', '')}),
|
||||
"prompt": ("STRING", {"multiline": True, "default": "A stunning photograph of a majestic wolf howling at a full moon."}),
|
||||
"ratio": (cls.IMAGE_RATIO_OPTIONS, {"default": "1024:1024"}),
|
||||
"runway_api_version": ("STRING", {"multiline": False, "default": cls.DEFAULT_RUNWAY_API_VERSION}),
|
||||
},
|
||||
"optional": {
|
||||
"image1": ("IMAGE",),
|
||||
"image2": ("IMAGE",),
|
||||
"image3": ("IMAGE",),
|
||||
"tag1": ("STRING", {"multiline": False, "default": ""}), # Default to empty, let user decide
|
||||
"tag2": ("STRING", {"multiline": False, "default": ""}),
|
||||
"tag3": ("STRING", {"multiline": False, "default": ""}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 4294967295}), # 0 will mean NOT_GIVEN
|
||||
"public_figure_threshold": (["auto", "low", "NOT_GIVEN"], {"default": "NOT_GIVEN"}),
|
||||
"polling_interval": ("INT", {"default": 5, "min": 1, "max": 60}),
|
||||
"max_polling_attempts": ("INT", {"default": 36, "min": 1, "max": 120}), # Max 10 mins for 5s interval
|
||||
"timeout_sdk": ("FLOAT", {"default": 60.0, "min": 5.0, "max": 300.0, "step": 1.0}), # For SDK client
|
||||
"timeout_download": ("FLOAT", {"default": 60.0, "min": 5.0, "max": 300.0, "step": 1.0}), # For image download
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("image", "status_text")
|
||||
FUNCTION = "generate_image_from_script_logic"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def __init__(self):
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message: str):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
full_message = f"[FL_RunwayImageAPI] {timestamp}: {message}"
|
||||
print(full_message)
|
||||
self.log_messages.append(full_message) # Store full message with timestamp
|
||||
|
||||
def _create_error_image(self, error_message="API Failed", width=1024, height=1024) -> torch.Tensor:
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
try:
|
||||
font = ImageFont.truetype("arial.ttf", 24)
|
||||
except IOError:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
text_bbox = draw.textbbox((0,0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log(f"Created error image: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _process_tensor_to_pil(self, tensor: torch.Tensor, name="Image") -> Union[Image.Image, None]:
|
||||
if tensor is None: return None
|
||||
if len(tensor.shape) == 4 and tensor.shape[0] == 1:
|
||||
image_np = tensor[0].cpu().numpy()
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
return Image.fromarray(image_np)
|
||||
self._log(f"Error: {name} tensor format incorrect: {tensor.shape}")
|
||||
return None
|
||||
|
||||
def _pil_to_data_uri(self, pil_image: Image.Image, image_format="PNG") -> str:
|
||||
buffered = io.BytesIO()
|
||||
pil_image.save(buffered, format=image_format)
|
||||
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
return f"data:image/{image_format.lower()};base64,{img_str}"
|
||||
|
||||
def generate_image_from_script_logic(
|
||||
self, api_key: str, prompt: str, ratio: str, runway_api_version: str,
|
||||
image1: torch.Tensor = None, image2: torch.Tensor = None, image3: torch.Tensor = None,
|
||||
tag1: str = "", tag2: str = "", tag3: str = "",
|
||||
seed: int = 0, public_figure_threshold: str = "NOT_GIVEN",
|
||||
polling_interval: int = 5, max_polling_attempts: int = 36,
|
||||
timeout_sdk: float = 60.0, timeout_download: float = 60.0
|
||||
):
|
||||
self.log_messages = [] # Reset logs for each run
|
||||
|
||||
if not api_key or not api_key.strip():
|
||||
self._log("Error: RunwayML API key not provided.")
|
||||
return self._create_error_image("API key missing"), "\n".join(self.log_messages)
|
||||
|
||||
# Use runway_api_version from input, or default if empty
|
||||
current_runway_api_version = runway_api_version if runway_api_version and runway_api_version.strip() else self.DEFAULT_RUNWAY_API_VERSION
|
||||
|
||||
try:
|
||||
client = RunwayML(api_key=api_key.strip(), runway_version=current_runway_api_version, timeout=timeout_sdk)
|
||||
self._log(f"RunwayML client initialized. API Version: {client.runway_version}, Timeout: {timeout_sdk}s")
|
||||
except Exception as e:
|
||||
self._log(f"Failed to initialize RunwayML client: {str(e)}")
|
||||
return self._create_error_image("SDK Client Init Failed"), "\n".join(self.log_messages)
|
||||
|
||||
active_task_id = None
|
||||
|
||||
reference_images_payload: List[Dict[str, Any]] = []
|
||||
input_images_with_tags = [(image1, tag1), (image2, tag2), (image3, tag3)]
|
||||
for i, (img_tensor, tag_str) in enumerate(input_images_with_tags):
|
||||
if img_tensor is not None:
|
||||
pil_img = self._process_tensor_to_pil(img_tensor, f"Reference Image {i+1}")
|
||||
if pil_img:
|
||||
data_uri = self._pil_to_data_uri(pil_img)
|
||||
ref_img_obj = {"uri": data_uri}
|
||||
clean_tag = tag_str.strip() if tag_str else ""
|
||||
if clean_tag and 3 <= len(clean_tag) <= 16 and clean_tag[0].isalpha() and clean_tag.replace('_', '').isalnum():
|
||||
ref_img_obj["tag"] = clean_tag
|
||||
elif clean_tag:
|
||||
self._log(f"Warning: Tag '{clean_tag}' for image {i+1} is invalid and will be ignored.")
|
||||
reference_images_payload.append(ref_img_obj)
|
||||
else:
|
||||
self._log(f"Error processing reference image {i+1}")
|
||||
return self._create_error_image(f"Failed to process ref image {i+1}"), "\n".join(self.log_messages)
|
||||
|
||||
create_args: Dict[str, Any] = {
|
||||
"model": self.MODEL_NAME,
|
||||
"prompt_text": prompt,
|
||||
"ratio": ratio,
|
||||
}
|
||||
if reference_images_payload: # Only add if there are valid reference images
|
||||
create_args["reference_images"] = reference_images_payload
|
||||
|
||||
current_seed = NOT_GIVEN if seed == 0 else seed
|
||||
create_args["seed"] = current_seed
|
||||
|
||||
if public_figure_threshold != "NOT_GIVEN":
|
||||
create_args["content_moderation"] = {"publicFigureThreshold": public_figure_threshold}
|
||||
|
||||
try:
|
||||
self._log(f"Requesting image generation with prompt: '{prompt}' using model '{self.MODEL_NAME}'")
|
||||
self._log(f"Create args (excluding images for brevity if many): { {k:v for k,v in create_args.items() if k != 'reference_images'} }")
|
||||
if 'reference_images' in create_args:
|
||||
self._log(f"Number of reference images: {len(create_args['reference_images'])}")
|
||||
|
||||
|
||||
initial_response = client.text_to_image.create(**create_args)
|
||||
active_task_id = initial_response.id
|
||||
self._log(f"Task submitted successfully! Task ID: {active_task_id}")
|
||||
|
||||
self._log(f"Polling task status for Task ID: {active_task_id} (every {polling_interval}s)...")
|
||||
task_details: Union[TaskRetrieveResponse, None] = None
|
||||
pbar = ProgressBar(max_polling_attempts) # Initialize ProgressBar
|
||||
for attempt in range(max_polling_attempts):
|
||||
task_details = client.tasks.retrieve(id=active_task_id)
|
||||
progress_percent = 0.0
|
||||
if task_details.status == "RUNNING" and task_details.progress is not None:
|
||||
progress_percent = task_details.progress * 100
|
||||
|
||||
status_log = f"Attempt {attempt + 1}/{max_polling_attempts} - Status: {task_details.status}"
|
||||
if progress_percent > 0:
|
||||
status_log += f", Progress: {progress_percent:.2f}%"
|
||||
self._log(status_log)
|
||||
|
||||
# Update progress bar: current value is attempt number, max is max_polling_attempts
|
||||
# If RUNNING and progress is available, use that for a more granular bar update.
|
||||
# However, ProgressBar typically expects an iteration count.
|
||||
# We can simulate this by updating based on attempts, or if RUNNING, map task_details.progress to pbar's scale.
|
||||
# For simplicity, we'll update based on attempts.
|
||||
# If task_details.progress is available, we can use it to show a more fine-grained update
|
||||
# by mapping it to the total attempts.
|
||||
# For now, just update based on attempt count.
|
||||
# A more sophisticated approach might involve setting pbar total to 100 and updating with progress_percent
|
||||
# but that might look weird if the task finishes early or takes many polls without progress change.
|
||||
|
||||
current_progress_for_bar = attempt + 1 # Default to attempt number
|
||||
if task_details.status == "RUNNING" and task_details.progress is not None:
|
||||
# Map task progress (0-1) to polling attempts for a smoother bar
|
||||
# This is an approximation. If progress jumps, bar will jump.
|
||||
# If progress is slow, bar updates per poll.
|
||||
current_progress_for_bar = int(task_details.progress * max_polling_attempts)
|
||||
# Ensure it doesn't exceed max_polling_attempts due to rounding or jumps
|
||||
current_progress_for_bar = min(current_progress_for_bar, max_polling_attempts)
|
||||
# Ensure it's at least the current attempt, so it doesn't go backwards
|
||||
current_progress_for_bar = max(current_progress_for_bar, attempt + 1)
|
||||
|
||||
# Temporarily comment out to diagnose AttributeError
|
||||
# pbar.update_absolute(current_progress_for_bar, max_polling_attempts, status_log)
|
||||
|
||||
|
||||
if task_details.status == "SUCCEEDED":
|
||||
self._log("Task SUCCEEDED!")
|
||||
if task_details.output and len(task_details.output) > 0 and isinstance(task_details.output[0], str):
|
||||
image_url = task_details.output[0]
|
||||
self._log(f"Image URL: {image_url}")
|
||||
self._log(f"Downloading image...")
|
||||
try:
|
||||
with httpx.Client() as download_client:
|
||||
response = download_client.get(image_url, follow_redirects=True, timeout=timeout_download)
|
||||
response.raise_for_status()
|
||||
pil_image = Image.open(io.BytesIO(response.content)).convert("RGB")
|
||||
img_array = np.array(pil_image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
self._log("Image downloaded and processed successfully.")
|
||||
return img_tensor, "\n".join(self.log_messages)
|
||||
except httpx.HTTPStatusError as e_http:
|
||||
self._log(f"Error downloading image: HTTP {e_http.response.status_code} - {e_http.response.text}")
|
||||
return self._create_error_image(f"Download Failed: {e_http.response.status_code}"), "\n".join(self.log_messages)
|
||||
except Exception as e_dl:
|
||||
self._log(f"An error occurred during image download: {str(e_dl)}")
|
||||
return self._create_error_image("Download Error"), "\n".join(self.log_messages)
|
||||
else:
|
||||
self._log("Task succeeded but no valid output URL found.")
|
||||
return self._create_error_image("No Output URL"), "\n".join(self.log_messages)
|
||||
|
||||
elif task_details.status in ["FAILED", "CANCELLED"]: # Your script uses CANCELLED, SDK might use CANCELED
|
||||
self._log(f"Task {task_details.status}.")
|
||||
failure_reason = task_details.failure if hasattr(task_details, 'failure') and task_details.failure else 'N/A'
|
||||
failure_code = task_details.failure_code if hasattr(task_details, 'failure_code') and task_details.failure_code else 'N/A'
|
||||
self._log(f" Reason: {failure_reason}")
|
||||
self._log(f" Failure code: {failure_code}")
|
||||
# ... (additional suggestions from your script can be added here if desired)
|
||||
return self._create_error_image(f"Task {task_details.status}: {failure_reason[:60]}"), "\n".join(self.log_messages)
|
||||
|
||||
time.sleep(polling_interval)
|
||||
|
||||
# If loop finishes without returning
|
||||
if task_details:
|
||||
self._log(f"Max poll attempts ({max_polling_attempts}) reached. Last known task status: {task_details.status}.")
|
||||
else: # Should not happen if initial submission was successful
|
||||
self._log(f"Max poll attempts ({max_polling_attempts}) reached. Could not retrieve task details after submission.")
|
||||
return self._create_error_image("Polling Timeout"), "\n".join(self.log_messages)
|
||||
|
||||
except APIConnectionError as e:
|
||||
self._log(f"RunwayML API Connection Error: {str(e)}")
|
||||
return self._create_error_image("API Connection Error"), "\n".join(self.log_messages)
|
||||
except RateLimitError as e:
|
||||
self._log(f"RunwayML Rate Limit Error: {str(e)}")
|
||||
return self._create_error_image("Rate Limit Error"), "\n".join(self.log_messages)
|
||||
except APIStatusError as e:
|
||||
self._log(f"RunwayML API Status Error: {str(e)}. Status Code: {e.status_code if hasattr(e, 'status_code') else 'N/A'}. Response: {e.response.text if hasattr(e, 'response') and hasattr(e.response, 'text') else 'N/A'}")
|
||||
return self._create_error_image(f"API Status Error: {e.status_code if hasattr(e, 'status_code') else ''}"), "\n".join(self.log_messages)
|
||||
except APIError as e: # General SDK error
|
||||
self._log(f"RunwayML API Error: {str(e)}")
|
||||
return self._create_error_image("Runway API Error"), "\n".join(self.log_messages)
|
||||
except Exception as e:
|
||||
self._log(f"An unexpected error occurred: {str(e)}")
|
||||
if active_task_id:
|
||||
self._log(f"This error occurred for Task ID: {active_task_id}")
|
||||
return self._create_error_image("Unexpected Node Error"), "\n".join(self.log_messages)
|
||||
finally:
|
||||
if 'client' in locals() and client is not None and hasattr(client, 'close'):
|
||||
try:
|
||||
client.close()
|
||||
self._log("RunwayML client closed.")
|
||||
except Exception as e_close:
|
||||
self._log(f"Error closing RunwayML client: {str(e_close)}")
|
||||
@@ -1,361 +0,0 @@
|
||||
import os
|
||||
import io
|
||||
import json
|
||||
import base64
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import time
|
||||
import traceback
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from google.oauth2 import service_account
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
class FL_VertexGemini25FlashImage:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt": ("STRING", {"multiline": True, "default": ""}),
|
||||
"service_account_json": ("STRING", {"default": "", "multiline": False}),
|
||||
"batch_count": ("INT", {"default": 1, "min": 1, "max": 8, "step": 1}),
|
||||
"temperature": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.1}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 666666}),
|
||||
},
|
||||
"optional": {
|
||||
"reference_image_1": ("IMAGE",),
|
||||
"reference_image_2": ("IMAGE",),
|
||||
"reference_image_3": ("IMAGE",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING")
|
||||
RETURN_NAMES = ("images", "API_Response")
|
||||
FUNCTION = "generate_image"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize logging system"""
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message):
|
||||
"""Global logging function: record to log list"""
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_VertexGemini25FlashImage] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _process_tensor_to_pil(self, tensor, name="Image"):
|
||||
"""Convert a tensor to a PIL image for API submission"""
|
||||
try:
|
||||
if tensor is None:
|
||||
self._log(f"{name} is None, skipping")
|
||||
return None
|
||||
|
||||
# Ensure tensor is in correct format [1, H, W, 3]
|
||||
if len(tensor.shape) == 4 and tensor.shape[0] >= 1:
|
||||
# Get first frame image
|
||||
image_np = tensor[0].cpu().numpy()
|
||||
|
||||
# Convert to uint8 format for PIL
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
|
||||
# Create PIL image
|
||||
pil_image = Image.fromarray(image_np)
|
||||
|
||||
self._log(f"{name} processed successfully, size: {pil_image.width}x{pil_image.height}")
|
||||
return pil_image
|
||||
else:
|
||||
self._log(f"{name} format incorrect: {tensor.shape}")
|
||||
return None
|
||||
except Exception as e:
|
||||
self._log(f"Error processing {name}: {str(e)}")
|
||||
return None
|
||||
|
||||
def _prepare_image_for_api(self, pil_image):
|
||||
"""Prepare PIL image for Vertex AI - just return the PIL image directly"""
|
||||
try:
|
||||
self._log(f"Preparing image for API: {pil_image.width}x{pil_image.height}")
|
||||
|
||||
# For Vertex AI generate_content with Gemini models,
|
||||
# we can pass PIL images directly and the SDK handles conversion
|
||||
return pil_image
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error preparing image: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def _create_error_frame(self, error_message="Image Generation Failed", width=512, height=512):
|
||||
"""Create a black error frame with red text"""
|
||||
from PIL import ImageDraw, ImageFont
|
||||
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
try:
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Draw error text
|
||||
text_bbox = draw.textbbox((0, 0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
|
||||
# Convert to tensor format [1, H, W, 3]
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
|
||||
self._log(f"Created error frame with message: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def generate_image(self, prompt: str, service_account_json: str,
|
||||
batch_count: int = 1,
|
||||
temperature: float = 1.0, seed: int = 0, reference_image_1=None,
|
||||
reference_image_2=None, reference_image_3=None):
|
||||
"""Generate image using Gemini 2.5 Flash Image API"""
|
||||
|
||||
# Reset log messages
|
||||
self.log_messages = []
|
||||
|
||||
# Log seed parameter (not sent to API, just for node tracking)
|
||||
self._log(f"Seed parameter: {seed} (not sent to API)")
|
||||
|
||||
try:
|
||||
# Validate service account JSON file
|
||||
if not service_account_json:
|
||||
error_message = "Error: No service account JSON file provided."
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("Service account required")
|
||||
return (error_frame, error_message)
|
||||
|
||||
# Build full path to service account JSON
|
||||
script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
json_path = os.path.join(script_dir, service_account_json)
|
||||
|
||||
if not os.path.exists(json_path):
|
||||
error_message = f"Error: Service account JSON file not found at {json_path}"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("JSON file not found")
|
||||
return (error_frame, error_message)
|
||||
|
||||
# Validate prompt
|
||||
if not prompt or len(prompt.strip()) == 0:
|
||||
error_message = "Error: No prompt provided. Please enter an image generation prompt."
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("Prompt required")
|
||||
return (error_frame, error_message)
|
||||
|
||||
# Load credentials from service account JSON
|
||||
self._log(f"Loading service account credentials from: {json_path}")
|
||||
|
||||
# Read the JSON file to get project_id
|
||||
with open(json_path, 'r') as f:
|
||||
service_account_info = json.load(f)
|
||||
project_id = service_account_info.get('project_id')
|
||||
|
||||
if not project_id:
|
||||
error_message = "Error: No project_id found in service account JSON"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("Missing project_id")
|
||||
return (error_frame, error_message)
|
||||
|
||||
credentials = service_account.Credentials.from_service_account_file(
|
||||
json_path,
|
||||
scopes=['https://www.googleapis.com/auth/cloud-platform']
|
||||
)
|
||||
|
||||
# Initialize client with Vertex AI
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=project_id,
|
||||
location='us-central1',
|
||||
credentials=credentials
|
||||
)
|
||||
self._log(f"Gemini client initialized with Vertex AI (project: {project_id})")
|
||||
|
||||
# Build the contents for the API call
|
||||
contents = []
|
||||
|
||||
# Add reference images if provided (up to 3)
|
||||
reference_images = [reference_image_1, reference_image_2, reference_image_3]
|
||||
image_count = 0
|
||||
|
||||
for i, ref_image in enumerate(reference_images, 1):
|
||||
if ref_image is not None:
|
||||
pil_reference = self._process_tensor_to_pil(ref_image, f"Reference Image {i}")
|
||||
if pil_reference:
|
||||
# Prepare image for Vertex AI
|
||||
image_for_api = self._prepare_image_for_api(pil_reference)
|
||||
if not image_for_api:
|
||||
error_message = f"Failed to prepare reference image {i}"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, error_message)
|
||||
contents.append(image_for_api)
|
||||
image_count += 1
|
||||
self._log(f"Reference image {i} added to request")
|
||||
|
||||
if image_count > 0:
|
||||
self._log(f"Total reference images: {image_count}")
|
||||
|
||||
# Add the prompt
|
||||
contents.append(prompt)
|
||||
|
||||
# Build the configuration
|
||||
# Note: Vertex AI doesn't support aspect_ratio in current version
|
||||
# Users can specify desired aspect ratio in the prompt text itself
|
||||
# candidate_count is hardcoded to 1, use batch_count for multiple images
|
||||
config = types.GenerateContentConfig(
|
||||
response_modalities=["IMAGE"],
|
||||
candidate_count=1,
|
||||
temperature=temperature
|
||||
)
|
||||
|
||||
self._log(f"Generating {batch_count} image(s) with prompt: {prompt[:100]}...")
|
||||
self._log(f"Temperature: {temperature}")
|
||||
|
||||
# Function to make a single API call
|
||||
def generate_single_image(batch_idx):
|
||||
try:
|
||||
self._log(f"Starting batch {batch_idx + 1}/{batch_count}...")
|
||||
|
||||
response = client.models.generate_content(
|
||||
model="gemini-2.5-flash-image",
|
||||
contents=contents,
|
||||
config=config
|
||||
)
|
||||
|
||||
self._log(f"Batch {batch_idx + 1} API response received")
|
||||
|
||||
# Log response structure for debugging
|
||||
self._log(f"Batch {batch_idx + 1} - Number of candidates: {len(response.candidates) if hasattr(response, 'candidates') else 'N/A'}")
|
||||
|
||||
# Check for prompt feedback (content filtering, safety blocks, etc.)
|
||||
if hasattr(response, 'prompt_feedback') and response.prompt_feedback:
|
||||
self._log(f"Batch {batch_idx + 1} - Prompt feedback: {response.prompt_feedback}")
|
||||
|
||||
# Extract image from response
|
||||
image_found = False
|
||||
for candidate_idx, candidate in enumerate(response.candidates):
|
||||
self._log(f"Batch {batch_idx + 1} - Candidate {candidate_idx}: {len(candidate.content.parts) if hasattr(candidate, 'content') else 'N/A'} parts")
|
||||
|
||||
# Check for finish reason (safety blocks, etc.)
|
||||
if hasattr(candidate, 'finish_reason'):
|
||||
self._log(f"Batch {batch_idx + 1} - Candidate {candidate_idx} finish_reason: {candidate.finish_reason}")
|
||||
|
||||
# Check for safety ratings
|
||||
if hasattr(candidate, 'safety_ratings') and candidate.safety_ratings:
|
||||
self._log(f"Batch {batch_idx + 1} - Candidate {candidate_idx} safety_ratings: {candidate.safety_ratings}")
|
||||
|
||||
for part_idx, part in enumerate(candidate.content.parts):
|
||||
self._log(f"Batch {batch_idx + 1} - Part {part_idx} type: {type(part).__name__}")
|
||||
self._log(f"Batch {batch_idx + 1} - Part {part_idx} has inline_data: {hasattr(part, 'inline_data')}")
|
||||
|
||||
if hasattr(part, 'inline_data') and part.inline_data:
|
||||
image_data = part.inline_data.data
|
||||
self._log(f"Batch {batch_idx + 1} - inline_data size: {len(image_data)} bytes")
|
||||
|
||||
# Check mime type if available
|
||||
if hasattr(part.inline_data, 'mime_type'):
|
||||
self._log(f"Batch {batch_idx + 1} - inline_data mime_type: {part.inline_data.mime_type}")
|
||||
|
||||
# Check if data is base64 encoded (starts with base64 characters)
|
||||
# Raw binary images start with magic bytes (e.g., PNG: \x89PNG, JPEG: \xff\xd8)
|
||||
# Base64 encoded data will be ASCII/UTF-8 text
|
||||
is_base64 = False
|
||||
try:
|
||||
# Check if the data looks like base64 (ASCII text)
|
||||
if isinstance(image_data, bytes):
|
||||
# Try to decode as ASCII - base64 is ASCII
|
||||
test_str = image_data[:20].decode('ascii')
|
||||
# Check if it starts with common base64 PNG prefix
|
||||
if test_str.startswith('iVBORw0KG') or test_str.startswith('/9j/'):
|
||||
is_base64 = True
|
||||
self._log(f"Batch {batch_idx + 1} - Detected base64-encoded data")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Decode base64 if needed
|
||||
if is_base64:
|
||||
try:
|
||||
image_data = base64.b64decode(image_data)
|
||||
self._log(f"Batch {batch_idx + 1} - Decoded base64, new size: {len(image_data)} bytes")
|
||||
except Exception as e:
|
||||
self._log(f"Batch {batch_idx + 1} - Failed to decode base64: {str(e)}")
|
||||
|
||||
# Check image header to identify format (after potential base64 decode)
|
||||
image_header = image_data[:16] if len(image_data) >= 16 else image_data
|
||||
self._log(f"Batch {batch_idx + 1} - image header (hex): {image_header.hex()}")
|
||||
|
||||
pil_image = Image.open(io.BytesIO(image_data))
|
||||
self._log(f"Batch {batch_idx + 1} image received: {pil_image.width}x{pil_image.height}")
|
||||
|
||||
# Convert to tensor format [H, W, 3]
|
||||
img_array = np.array(pil_image).astype(np.float32) / 255.0
|
||||
|
||||
# Handle RGBA images
|
||||
if img_array.shape[-1] == 4:
|
||||
img_array = img_array[:, :, :3]
|
||||
|
||||
image_found = True
|
||||
return torch.from_numpy(img_array)
|
||||
elif hasattr(part, 'text'):
|
||||
self._log(f"Batch {batch_idx + 1} - Part {part_idx} contains text: {part.text[:100] if len(part.text) > 100 else part.text}")
|
||||
|
||||
if not image_found:
|
||||
self._log(f"Batch {batch_idx + 1} - WARNING: No image data found in response")
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
self._log(f"Error in batch {batch_idx + 1}: {str(e)}")
|
||||
self._log(f"Batch {batch_idx + 1} - Full traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
# Generate images in parallel using ThreadPoolExecutor
|
||||
generated_images = []
|
||||
with ThreadPoolExecutor(max_workers=batch_count) as executor:
|
||||
futures = [executor.submit(generate_single_image, i) for i in range(batch_count)]
|
||||
for future in futures:
|
||||
result = future.result()
|
||||
if result is not None:
|
||||
generated_images.append(result)
|
||||
|
||||
if not generated_images:
|
||||
error_message = "No images generated by the API"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, error_message)
|
||||
|
||||
# Stack images into a batch tensor [B, H, W, C]
|
||||
images_batch = torch.stack(generated_images, dim=0)
|
||||
|
||||
# Prepare response text
|
||||
response_text = "## Image Generation Successful\n"
|
||||
response_text += f"Generated {len(generated_images)} image(s)\n"
|
||||
response_text += f"Batch shape: {images_batch.shape}\n"
|
||||
response_text += "\n## Processing Log\n" + "\n".join(self.log_messages)
|
||||
|
||||
self._log(f"Successfully generated {len(generated_images)} image(s)")
|
||||
|
||||
return (images_batch, response_text)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Error during image generation: {str(e)}"
|
||||
self._log(error_message)
|
||||
traceback.print_exc()
|
||||
|
||||
error_frame = self._create_error_frame(f"Error: {str(e)}")
|
||||
full_text = "## Processing Log\n" + "\n".join(self.log_messages) + "\n\n## Error\n" + error_message
|
||||
|
||||
return (error_frame, full_text)
|
||||
@@ -1,713 +0,0 @@
|
||||
import os
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import requests
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
from google.auth import default
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import service_account
|
||||
import urllib.request
|
||||
from typing import Optional, Tuple
|
||||
|
||||
|
||||
class FL_Veo3VideoGen:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt": ("STRING", {"multiline": True, "default": ""}),
|
||||
"service_account_json": ("STRING", {"default": "", "multiline": False}),
|
||||
"model": (
|
||||
[
|
||||
"veo-3.0-generate-001",
|
||||
"veo-3.0-fast-generate-001",
|
||||
"veo-2.0-generate-001"
|
||||
],
|
||||
{"default": "veo-3.0-generate-001"}
|
||||
),
|
||||
"aspect_ratio": (
|
||||
["16:9", "9:16"],
|
||||
{"default": "16:9"}
|
||||
),
|
||||
"resolution": (
|
||||
["720p", "1080p"],
|
||||
{"default": "720p"}
|
||||
),
|
||||
"max_retries": ("INT", {"default": 3, "min": 1, "max": 10, "step": 1}),
|
||||
"polling_interval": ("INT", {"default": 10, "min": 5, "max": 60, "step": 5, "description": "Seconds between status checks"}),
|
||||
"max_wait_time": ("INT", {"default": 360, "min": 60, "max": 600, "step": 30, "description": "Maximum seconds to wait for video generation"}),
|
||||
},
|
||||
"optional": {
|
||||
"negative_prompt": ("STRING", {"multiline": True, "default": ""}),
|
||||
"seed": ("INT", {"default": 0, "min": 0, "max": 2147483647}),
|
||||
"reference_image": ("IMAGE",),
|
||||
"enable_person_generation": ("BOOLEAN", {"default": True}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE", "STRING", "STRING")
|
||||
RETURN_NAMES = ("frames", "video_path", "API_Response")
|
||||
FUNCTION = "generate_video"
|
||||
CATEGORY = "🏵️Fill Nodes/AI"
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize logging system"""
|
||||
self.log_messages = []
|
||||
|
||||
def _log(self, message):
|
||||
"""Global logging function: record to log list"""
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
formatted_message = f"[FL_Veo3VideoGen] {timestamp}: {message}"
|
||||
print(formatted_message)
|
||||
self.log_messages.append(message)
|
||||
return message
|
||||
|
||||
def _process_tensor_to_pil(self, tensor, name="Image"):
|
||||
"""Convert a tensor to a PIL image for API submission"""
|
||||
try:
|
||||
if tensor is None:
|
||||
self._log(f"{name} is None, skipping")
|
||||
return None
|
||||
|
||||
# Ensure tensor is in correct format [1, H, W, 3]
|
||||
if len(tensor.shape) == 4 and tensor.shape[0] >= 1:
|
||||
# Get first frame image
|
||||
image_np = tensor[0].cpu().numpy()
|
||||
|
||||
# Convert to uint8 format for PIL
|
||||
image_np = (image_np * 255).astype(np.uint8)
|
||||
|
||||
# Create PIL image
|
||||
pil_image = Image.fromarray(image_np)
|
||||
|
||||
self._log(f"{name} processed successfully, size: {pil_image.width}x{pil_image.height}")
|
||||
return pil_image
|
||||
else:
|
||||
self._log(f"{name} format incorrect: {tensor.shape}")
|
||||
return None
|
||||
except Exception as e:
|
||||
self._log(f"Error processing {name}: {str(e)}")
|
||||
return None
|
||||
|
||||
def _prepare_image_for_api(self, pil_image):
|
||||
"""Prepare PIL image for Vertex AI"""
|
||||
try:
|
||||
from google.genai.types import Image as GenAIImage
|
||||
|
||||
self._log(f"Preparing image for API: {pil_image.width}x{pil_image.height}")
|
||||
|
||||
# Convert PIL image to bytes
|
||||
img_byte_arr = io.BytesIO()
|
||||
pil_image.save(img_byte_arr, format='PNG')
|
||||
img_bytes = img_byte_arr.getvalue()
|
||||
|
||||
self._log(f"Image converted to bytes: {len(img_bytes)} bytes")
|
||||
|
||||
# Create Image object with bytes
|
||||
image_obj = GenAIImage(
|
||||
image_bytes=img_bytes,
|
||||
mime_type="image/png"
|
||||
)
|
||||
|
||||
return image_obj
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error preparing image: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def _create_error_frame(self, error_message="Video Generation Failed", width=1280, height=720):
|
||||
"""Create a black error frame with red text"""
|
||||
from PIL import ImageDraw, ImageFont
|
||||
|
||||
image = Image.new('RGB', (width, height), color=(0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
try:
|
||||
font = ImageFont.load_default()
|
||||
except Exception:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Draw error text
|
||||
text_bbox = draw.textbbox((0, 0), error_message, font=font)
|
||||
text_width = text_bbox[2] - text_bbox[0]
|
||||
text_height = text_bbox[3] - text_bbox[1]
|
||||
text_x = (width - text_width) / 2
|
||||
text_y = (height - text_height) / 2
|
||||
|
||||
draw.text((text_x, text_y), error_message, fill=(255, 0, 0), font=font)
|
||||
|
||||
# Convert to tensor format [1, H, W, 3]
|
||||
img_array = np.array(image).astype(np.float32) / 255.0
|
||||
img_tensor = torch.from_numpy(img_array).unsqueeze(0)
|
||||
|
||||
self._log(f"Created error frame with message: '{error_message}'")
|
||||
return img_tensor
|
||||
|
||||
def _download_video(self, video_url: str) -> Optional[str]:
|
||||
"""Download video from URL to temporary file"""
|
||||
try:
|
||||
self._log(f"Downloading video from: {video_url}")
|
||||
|
||||
temp_file = os.path.join(
|
||||
tempfile.gettempdir(),
|
||||
f"veo3_video_{int(time.time())}.mp4"
|
||||
)
|
||||
|
||||
urllib.request.urlretrieve(video_url, temp_file)
|
||||
|
||||
file_size = os.path.getsize(temp_file)
|
||||
self._log(f"Video downloaded successfully: {file_size} bytes")
|
||||
|
||||
return temp_file
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error downloading video: {str(e)}")
|
||||
return None
|
||||
|
||||
def _extract_frames_from_video(self, video_path: str) -> list:
|
||||
"""Extract all frames from video file"""
|
||||
try:
|
||||
import cv2
|
||||
|
||||
self._log(f"Extracting frames from: {video_path}")
|
||||
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
frames = []
|
||||
|
||||
if not cap.isOpened():
|
||||
self._log("Error: Could not open video file")
|
||||
return []
|
||||
|
||||
frame_count = 0
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
# Convert BGR to RGB
|
||||
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
# Convert to tensor format [H, W, 3] with values 0-1
|
||||
frame_array = frame_rgb.astype(np.float32) / 255.0
|
||||
frame_tensor = torch.from_numpy(frame_array)
|
||||
|
||||
frames.append(frame_tensor)
|
||||
frame_count += 1
|
||||
|
||||
cap.release()
|
||||
|
||||
self._log(f"Extracted {frame_count} frames from video")
|
||||
|
||||
return frames
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error extracting frames: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return []
|
||||
|
||||
def _call_veo_api(self, client, model: str, prompt: str, config,
|
||||
reference_image = None,
|
||||
retry_count: int = 0, max_retries: int = 3) -> Optional[any]:
|
||||
"""Call VEO API with retry logic"""
|
||||
try:
|
||||
self._log(f"API call attempt #{retry_count + 1}")
|
||||
self._log(f"Model: {model}")
|
||||
self._log(f"Prompt: {prompt[:100]}..." if len(prompt) > 100 else f"Prompt: {prompt}")
|
||||
|
||||
# Make the API call using generate_videos
|
||||
if reference_image is not None:
|
||||
self._log(f"Using reference image")
|
||||
if config is not None:
|
||||
operation = client.models.generate_videos(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
image=reference_image,
|
||||
config=config
|
||||
)
|
||||
else:
|
||||
operation = client.models.generate_videos(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
image=reference_image
|
||||
)
|
||||
else:
|
||||
if config is not None:
|
||||
operation = client.models.generate_videos(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
config=config
|
||||
)
|
||||
else:
|
||||
operation = client.models.generate_videos(
|
||||
model=model,
|
||||
prompt=prompt
|
||||
)
|
||||
|
||||
self._log("Initial API response received")
|
||||
return operation
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"API call error: {str(e)}")
|
||||
if retry_count < max_retries - 1:
|
||||
wait_time = 5 * (retry_count + 1)
|
||||
self._log(f"Retrying in {wait_time} seconds... (Attempt {retry_count + 1}/{max_retries})")
|
||||
time.sleep(wait_time)
|
||||
return self._call_veo_api(client, model, prompt, config, reference_image,
|
||||
retry_count + 1, max_retries)
|
||||
else:
|
||||
self._log(f"Maximum retries ({max_retries}) reached. Giving up.")
|
||||
return None
|
||||
|
||||
def _poll_video_generation(self, client, operation,
|
||||
polling_interval: int = 10,
|
||||
max_wait_time: int = 360) -> Optional[dict]:
|
||||
"""Poll the API until video generation is complete"""
|
||||
try:
|
||||
start_time = time.time()
|
||||
self._log(f"Starting to poll for video generation")
|
||||
|
||||
while not operation.done:
|
||||
elapsed_time = time.time() - start_time
|
||||
|
||||
if elapsed_time > max_wait_time:
|
||||
self._log(f"Maximum wait time ({max_wait_time}s) exceeded")
|
||||
return None
|
||||
|
||||
self._log(f"Still processing... ({elapsed_time:.1f}s elapsed)")
|
||||
time.sleep(polling_interval)
|
||||
|
||||
# Refresh operation status
|
||||
try:
|
||||
operation = client.operations.get(operation)
|
||||
except Exception as e:
|
||||
self._log(f"Error checking operation status: {str(e)}")
|
||||
time.sleep(polling_interval)
|
||||
|
||||
elapsed_time = time.time() - start_time
|
||||
self._log(f"Video generation completed after {elapsed_time:.1f} seconds")
|
||||
return operation
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error during polling: {str(e)}")
|
||||
return None
|
||||
|
||||
def generate_video(self, prompt: str, service_account_json: str, model: str,
|
||||
aspect_ratio: str = "16:9", resolution: str = "720p",
|
||||
max_retries: int = 3, polling_interval: int = 10,
|
||||
max_wait_time: int = 360, negative_prompt: str = "",
|
||||
seed: int = 0, reference_image=None,
|
||||
enable_person_generation: bool = True):
|
||||
"""Generate video using VEO 3 API"""
|
||||
|
||||
# Reset log messages
|
||||
self.log_messages = []
|
||||
|
||||
try:
|
||||
# Validate service account JSON file
|
||||
if not service_account_json:
|
||||
error_message = "Error: No service account JSON file provided."
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("Service account required")
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
# Build full path to service account JSON
|
||||
script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
json_path = os.path.join(script_dir, service_account_json)
|
||||
|
||||
if not os.path.exists(json_path):
|
||||
error_message = f"Error: Service account JSON file not found at {json_path}"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("JSON file not found")
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
# Validate prompt
|
||||
if not prompt or len(prompt.strip()) == 0:
|
||||
error_message = "Error: No prompt provided. Please enter a video generation prompt."
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("Prompt required")
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
# Validate resolution for aspect ratio
|
||||
if resolution == "1080p" and aspect_ratio != "16:9":
|
||||
self._log("Warning: 1080p is only available for 16:9 aspect ratio. Defaulting to 720p.")
|
||||
resolution = "720p"
|
||||
|
||||
# Load credentials from service account JSON
|
||||
self._log(f"Loading service account credentials from: {json_path}")
|
||||
|
||||
# Read the JSON file to get project_id
|
||||
with open(json_path, 'r') as f:
|
||||
service_account_info = json.load(f)
|
||||
project_id = service_account_info.get('project_id')
|
||||
|
||||
if not project_id:
|
||||
error_message = "Error: No project_id found in service account JSON"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame("Missing project_id")
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
credentials = service_account.Credentials.from_service_account_file(
|
||||
json_path,
|
||||
scopes=['https://www.googleapis.com/auth/cloud-platform']
|
||||
)
|
||||
|
||||
# Initialize client with Vertex AI
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=project_id,
|
||||
location='us-central1',
|
||||
credentials=credentials
|
||||
)
|
||||
self._log(f"Gemini client initialized with Vertex AI (project: {project_id})")
|
||||
|
||||
# Store credentials for later use in downloads
|
||||
self._credentials = credentials
|
||||
|
||||
# Process reference image if provided
|
||||
image_for_api = None
|
||||
if reference_image is not None:
|
||||
pil_reference = self._process_tensor_to_pil(reference_image, "Reference Image")
|
||||
if pil_reference:
|
||||
# Prepare image for Vertex AI
|
||||
image_for_api = self._prepare_image_for_api(pil_reference)
|
||||
if not image_for_api:
|
||||
error_message = "Failed to prepare reference image"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
# Build generation config
|
||||
# Start with None and only add if basic call fails
|
||||
gen_config = None
|
||||
|
||||
self._log(f"Requested aspect ratio: {aspect_ratio}")
|
||||
self._log(f"Requested resolution: {resolution}")
|
||||
if negative_prompt:
|
||||
self._log(f"Negative prompt: {negative_prompt[:100]}...")
|
||||
if seed > 0:
|
||||
self._log(f"Note: Seed {seed} provided")
|
||||
self._log(f"Person generation: {'enabled' if enable_person_generation else 'disabled'}")
|
||||
|
||||
# Make initial API call
|
||||
self._log("Initiating video generation...")
|
||||
operation = self._call_veo_api(
|
||||
client=client,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
config=gen_config,
|
||||
reference_image=image_for_api,
|
||||
max_retries=max_retries
|
||||
)
|
||||
|
||||
if operation is None:
|
||||
error_message = "Failed to initiate video generation"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
self._log(f"Operation started, waiting for completion...")
|
||||
|
||||
# Poll for completion
|
||||
completed_operation = self._poll_video_generation(
|
||||
client=client,
|
||||
operation=operation,
|
||||
polling_interval=polling_interval,
|
||||
max_wait_time=max_wait_time
|
||||
)
|
||||
|
||||
if completed_operation is None:
|
||||
error_message = "Video generation timed out or failed"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
# Extract video from completed operation
|
||||
video_file = self._extract_video_from_operation(completed_operation)
|
||||
|
||||
if not video_file:
|
||||
error_message = "No video found in completed operation"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
return self._process_video_result(client, video_file)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Error during video generation: {str(e)}"
|
||||
self._log(error_message)
|
||||
traceback.print_exc()
|
||||
|
||||
error_frame = self._create_error_frame(f"Error: {str(e)}")
|
||||
full_text = "## Processing Log\n" + "\n".join(self.log_messages) + "\n\n## Error\n" + error_message
|
||||
|
||||
return (error_frame, "", full_text)
|
||||
|
||||
def _extract_video_from_operation(self, operation) -> Optional[any]:
|
||||
"""Extract video file object from completed operation"""
|
||||
try:
|
||||
# Try multiple possible structures
|
||||
|
||||
# Try operation.result first
|
||||
if hasattr(operation, 'result') and operation.result:
|
||||
self._log(f"Found result attribute")
|
||||
self._log(f"Result type: {type(operation.result)}")
|
||||
self._log(f"Result attributes: {dir(operation.result)}")
|
||||
|
||||
# Check for generated_videos in result
|
||||
if hasattr(operation.result, 'generated_videos') and operation.result.generated_videos:
|
||||
if len(operation.result.generated_videos) > 0:
|
||||
generated_video = operation.result.generated_videos[0]
|
||||
self._log(f"Generated video type: {type(generated_video)}")
|
||||
self._log(f"Generated video attributes: {dir(generated_video)}")
|
||||
if hasattr(generated_video, 'video'):
|
||||
self._log(f"Found video file object")
|
||||
return generated_video.video
|
||||
|
||||
# Maybe the result IS the video
|
||||
return operation.result
|
||||
|
||||
# Try operation.response
|
||||
if hasattr(operation, 'response') and operation.response:
|
||||
if hasattr(operation.response, 'generated_videos') and operation.response.generated_videos:
|
||||
if len(operation.response.generated_videos) > 0:
|
||||
generated_video = operation.response.generated_videos[0]
|
||||
if hasattr(generated_video, 'video'):
|
||||
self._log(f"Found video file object in response")
|
||||
return generated_video.video
|
||||
|
||||
self._log("Could not find video in operation structure")
|
||||
self._log(f"Operation type: {type(operation)}")
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Error extracting video from operation: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def _process_video_result(self, client, video_file) -> Tuple:
|
||||
"""Download and process the generated video"""
|
||||
try:
|
||||
self._log(f"Processing video result")
|
||||
self._log(f"Video file type: {type(video_file)}")
|
||||
|
||||
# Download video file using the client
|
||||
temp_file = os.path.join(
|
||||
tempfile.gettempdir(),
|
||||
f"veo3_video_{int(time.time())}.mp4"
|
||||
)
|
||||
|
||||
self._log(f"Attempting to save video to: {temp_file}")
|
||||
|
||||
# Check if video_bytes is available (Vertex AI)
|
||||
if hasattr(video_file, 'video_bytes') and video_file.video_bytes:
|
||||
self._log(f"Using video_bytes from Video object")
|
||||
|
||||
video_data = video_file.video_bytes
|
||||
self._log(f"video_bytes size: {len(video_data)} bytes")
|
||||
|
||||
# Check if data is base64 encoded
|
||||
# Raw MP4 files start with specific magic bytes
|
||||
# MP4/M4V: 0x66747970 ('ftyp') at offset 4
|
||||
# Base64 encoded data will be ASCII/UTF-8 text
|
||||
is_base64 = False
|
||||
try:
|
||||
if isinstance(video_data, bytes):
|
||||
# Check first 20 bytes
|
||||
test_header = video_data[:20]
|
||||
self._log(f"Video data header (hex): {test_header.hex()}")
|
||||
|
||||
# Try to decode as ASCII - base64 is ASCII
|
||||
try:
|
||||
test_str = test_header.decode('ascii')
|
||||
# Check if it looks like base64 (no control chars, valid base64 chars)
|
||||
# Common base64 patterns for video might start with various chars
|
||||
# But we can check if it's printable ASCII which binary MP4 is not
|
||||
if all(c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r' for c in test_str):
|
||||
is_base64 = True
|
||||
self._log(f"Detected base64-encoded video data")
|
||||
except UnicodeDecodeError:
|
||||
# Binary data, not base64
|
||||
self._log(f"Video data is binary (not base64)")
|
||||
pass
|
||||
except Exception as e:
|
||||
self._log(f"Error checking video data format: {str(e)}")
|
||||
|
||||
# Decode base64 if needed
|
||||
if is_base64:
|
||||
try:
|
||||
original_size = len(video_data)
|
||||
video_data = base64.b64decode(video_data)
|
||||
self._log(f"Decoded base64 video data: {original_size} -> {len(video_data)} bytes")
|
||||
|
||||
# Verify it's now a valid video file
|
||||
video_header = video_data[:20] if len(video_data) >= 20 else video_data
|
||||
self._log(f"Decoded video header (hex): {video_header.hex()}")
|
||||
except Exception as e:
|
||||
self._log(f"Failed to decode base64 video data: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return self._create_error_frame(f"Base64 decode error: {str(e)}"), "", str(e)
|
||||
|
||||
# Write the video data to file
|
||||
with open(temp_file, 'wb') as f:
|
||||
f.write(video_data)
|
||||
|
||||
file_size = os.path.getsize(temp_file)
|
||||
self._log(f"Video saved: {file_size} bytes")
|
||||
|
||||
# Verify the file is a valid MP4
|
||||
with open(temp_file, 'rb') as f:
|
||||
file_header = f.read(20)
|
||||
self._log(f"Saved file header (hex): {file_header.hex()}")
|
||||
# Check for MP4 signature (ftyp at offset 4-7)
|
||||
if len(file_header) >= 8:
|
||||
ftyp_check = file_header[4:8]
|
||||
if ftyp_check == b'ftyp':
|
||||
self._log(f"Verified valid MP4 file signature")
|
||||
else:
|
||||
self._log(f"WARNING: File may not be a valid MP4 (ftyp signature not found)")
|
||||
else:
|
||||
self._log(f"WARNING: File too small to verify MP4 signature")
|
||||
|
||||
# Otherwise try using the URI (Google AI API)
|
||||
elif hasattr(video_file, 'uri') and video_file.uri:
|
||||
self._log(f"Attempting to download from URI")
|
||||
# Extract file ID from URI
|
||||
# URI format: https://generativelanguage.googleapis.com/v1beta/files/{file_id}:download?alt=media
|
||||
try:
|
||||
file_id_match = video_file.uri.split('/files/')
|
||||
if len(file_id_match) > 1:
|
||||
file_id = file_id_match[1].split(':')[0]
|
||||
self._log(f"Extracted file ID: {file_id}")
|
||||
|
||||
# Try to download using the file ID with authenticated request
|
||||
try:
|
||||
# Refresh credentials if needed
|
||||
if not self._credentials.valid:
|
||||
self._credentials.refresh(Request())
|
||||
|
||||
# Get the access token from service account credentials
|
||||
download_url = f"https://generativelanguage.googleapis.com/v1beta/files/{file_id}"
|
||||
|
||||
headers = {
|
||||
'Authorization': f'Bearer {self._credentials.token}'
|
||||
}
|
||||
|
||||
self._log(f"Downloading from: {download_url}")
|
||||
|
||||
response = requests.get(download_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
# Get the actual download URL from the response
|
||||
file_info = response.json()
|
||||
self._log(f"File info: {file_info}")
|
||||
|
||||
# Check if file is still processing
|
||||
if file_info.get('state') == 'PROCESSING':
|
||||
self._log(f"File is still processing, waiting for it to be ready...")
|
||||
max_file_wait = 60 # Wait up to 60 seconds for file processing
|
||||
file_wait_start = time.time()
|
||||
|
||||
while file_info.get('state') == 'PROCESSING':
|
||||
if time.time() - file_wait_start > max_file_wait:
|
||||
self._log("File processing timeout")
|
||||
return self._create_error_frame("File processing timeout"), "", "Timeout"
|
||||
|
||||
time.sleep(2)
|
||||
response = requests.get(download_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
file_info = response.json()
|
||||
self._log(f"File state: {file_info.get('state')}")
|
||||
|
||||
self._log(f"File is ready, state: {file_info.get('state')}")
|
||||
|
||||
# Now download the actual video
|
||||
if 'downloadUri' in file_info:
|
||||
video_uri = file_info['downloadUri']
|
||||
self._log(f"Downloading video from downloadUri: {video_uri}")
|
||||
|
||||
video_response = requests.get(video_uri, headers=headers, stream=True)
|
||||
video_response.raise_for_status()
|
||||
|
||||
with open(temp_file, 'wb') as f:
|
||||
for chunk in video_response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(temp_file)
|
||||
self._log(f"Video saved: {file_size} bytes")
|
||||
elif 'uri' in file_info:
|
||||
video_uri = file_info['uri']
|
||||
self._log(f"Downloading video from: {video_uri}")
|
||||
|
||||
video_response = requests.get(video_uri, headers=headers, stream=True)
|
||||
video_response.raise_for_status()
|
||||
|
||||
with open(temp_file, 'wb') as f:
|
||||
for chunk in video_response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
file_size = os.path.getsize(temp_file)
|
||||
self._log(f"Video saved: {file_size} bytes")
|
||||
else:
|
||||
self._log("No URI in file info")
|
||||
return self._create_error_frame("No download URI"), "", "No URI"
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"Authenticated download failed: {e}")
|
||||
traceback.print_exc()
|
||||
raise
|
||||
except Exception as e:
|
||||
self._log(f"URI download failed: {e}")
|
||||
traceback.print_exc()
|
||||
return self._create_error_frame(f"Download error: {str(e)}"), "", str(e)
|
||||
else:
|
||||
self._log("No video_bytes or URI found in video object")
|
||||
return self._create_error_frame("No video data found"), "", "No video data"
|
||||
|
||||
video_path = temp_file
|
||||
|
||||
# Extract frames
|
||||
frames = self._extract_frames_from_video(video_path)
|
||||
|
||||
if not frames or len(frames) == 0:
|
||||
error_message = "Failed to extract frames from video"
|
||||
self._log(error_message)
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, "", error_message)
|
||||
|
||||
# Clean up temp file
|
||||
try:
|
||||
os.remove(video_path)
|
||||
self._log("Temporary video file cleaned up")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Stack frames into a batch tensor [B, H, W, C]
|
||||
frames_batch = torch.stack(frames, dim=0)
|
||||
|
||||
# Prepare response text
|
||||
response_text = "## Video Generation Successful\n"
|
||||
response_text += f"Video saved to: {video_path}\n"
|
||||
response_text += f"Total frames extracted: {len(frames)}\n"
|
||||
response_text += f"Batch shape: {frames_batch.shape}\n"
|
||||
response_text += "\n## Processing Log\n" + "\n".join(self.log_messages)
|
||||
|
||||
self._log(f"Successfully processed video with {len(frames)} frames")
|
||||
|
||||
return (frames_batch, video_path, response_text)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Error processing video result: {str(e)}"
|
||||
self._log(error_message)
|
||||
traceback.print_exc()
|
||||
error_frame = self._create_error_frame(error_message)
|
||||
return (error_frame, "", error_message)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user