feat: upload avatar avatar_main_output node

# Conflicts:
#	blender/avatar_main_output.py

.gitignore

@@ -1,7 +1,9 @@
 # Created by https://www.toptal.com/developers/gitignore/api/node,python,react
 # Edit at https://www.toptal.com/developers/gitignore?templates=node,python,react
 
+workflow_templates/
 js/output.css
+*.task
 
 ### Node ###
 # Logs

README.md

@@ -2,6 +2,8 @@
 
 ![image](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270574817-201a005b-7e00-4671-85a1-54937bf0704e.png)
 
+Wanna animate or got a question? Join our [Discord](https://discord.gg/Xp6mZ4Ez5P)
+
 A custom nodes module for **creating real-time interactive avatars** powered by blender bpy mesh api + Avatech Shape Flow runtime.
 
 > **WARNING**  
@@ -9,196 +11,159 @@ A custom nodes module for **creating real-time interactive avatars** powered by
 
 # Demo
 
-| <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/12e2bfc6-438e-4d16-bead-9957ced3bae1" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=cce15b92-6d1c-4966-91b9-362d7833cb5d) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/0c497025-7ed5-4e25-b4d1-5a257e1ba814" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=42a8182f-b140-48c0-a556-35cddf0f76f7) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/a2bf71e3-0d9c-4ddd-957f-a6b0cb7e622a" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=7c23b8d6-d1a5-41c7-a084-250461dbef22) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/ad808c42-5297-4e61-8be8-d5cb7729d2ff" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=268b32c4-f9b9-4db8-a27c-a7e974f0f0ac) |
-|:---:|:---:|:---:|:---:|
-| <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/1d1ad8f9-31a6-48ec-bad2-ce972ee3b12f" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=f97fc5bb-93b0-4b02-bbc0-327dd41d0fc5) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/06958585-f780-4b38-8f5d-bddabd7da78a" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=4d50aa03-26e4-47e7-97b6-c3fe9d8fc96e) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/3d0e6b54-d45f-45ac-90bd-d8b149880f98" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=791014cb-7836-4641-afdb-ac331064b682) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/1d1ad8f9-31a6-48ec-bad2-ce972ee3b12f" width="186"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=f97fc5bb-93b0-4b02-bbc0-327dd41d0fc5) |
-
+| <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/12e2bfc6-438e-4d16-bead-9957ced3bae1" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=cce15b92-6d1c-4966-91b9-362d7833cb5d) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/0c497025-7ed5-4e25-b4d1-5a257e1ba814" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=42a8182f-b140-48c0-a556-35cddf0f76f7) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/a2bf71e3-0d9c-4ddd-957f-a6b0cb7e622a" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=7c23b8d6-d1a5-41c7-a084-250461dbef22) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/ad808c42-5297-4e61-8be8-d5cb7729d2ff" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=268b32c4-f9b9-4db8-a27c-a7e974f0f0ac) |
+| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
+| <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/1d1ad8f9-31a6-48ec-bad2-ce972ee3b12f" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=f97fc5bb-93b0-4b02-bbc0-327dd41d0fc5) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/06958585-f780-4b38-8f5d-bddabd7da78a" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=4d50aa03-26e4-47e7-97b6-c3fe9d8fc96e) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/3d0e6b54-d45f-45ac-90bd-d8b149880f98" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=791014cb-7836-4641-afdb-ac331064b682) | <img src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/1d1ad8f9-31a6-48ec-bad2-ce972ee3b12f" width="180"/><br>[Interact 👆](https://editor.avatech.ai/viewer?avatarId=f97fc5bb-93b0-4b02-bbc0-327dd41d0fc5) |
 
 # How to?
-- [Basic Workflow Template](#basic-workflow-template)
-- [Best Practices for image input](#image-preprocess-guide)
+
+- [Basic Rigging Workflow Template](#basic-rigging-workflow-template)
+- [Best Practices for image input](#best-practices-for-image-input)
 - [Custom Nodes List](#custom-nodes)
 - [Shape Flow](#shape-flow)
 - [Installation](#installation)
 - [Development](#development)
 - [Join Discord 💬](https://discord.gg/WNtBYksDwF)
 
-# Basic Workflow Template
+# Basic Rigging Workflow Template
 
-### Creating an eye blink and lipsync avatar
+### 1. Creating an eye blink and lipsync avatar
 
-> For optimal results, please input a character image with an open mouth and a minimum resolution of 768x768. This higher resolution will enable the tool to accurately recognize and work with facial features.
-> 
-> [💡Make your character mouth open Guide](#mouth-open-guide-inpaint) [💡Generate new image Guide](#character-gen-prompting-guide)
+![ComfyUI_00668_](https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/72a0abe8-2482-4a0d-8436-8eb231fd2f6d)
 
-![eye+mouth movement](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270634138-8a237b9d-05fc-4e4a-b802-6465911f0d77.png)
-Download: 📂[Template01 - Simple Shape Flow](https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/SimpleEye+MouthMovement.json)
+For optimal results, please input a character image with an open mouth and a minimum resolution of 768x768. This higher resolution will enable the tool to accurately recognize and work with facial features.
 
-Download: 📂[Template01 - ControlNet Gen](https://github.com/avatechai/avatar-graph-comfyui/tree/main/workflow_templates/TemplateGen01)
-_If you don't want to modify any values in the custom nodes, you can download the ControlNet Gen Template to generate your own image._
+Download: Save the image, and drag into Comfyui or [Simple Shape Flow](https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/SimpleEye+MouthMovement.json)
 
-<details> 
-<summary> Template01 - Nodes Value Setting Guide </summary>
+### 2. Creating an eye blink and lipsync emoji avatar
 
-## Template01 - Nodes Value Setting Guide
+| ![ComfyUI_00045_](https://github.com/avatechai/avatar-graph-comfyui/assets/18395202/b4787166-85df-43c6-9fe9-252989f68d18) | ![emoji_480p30_high](https://github.com/avatechai/avatar-graph-comfyui/assets/18395202/7d8b2b0a-e979-421d-8055-b4acac50a0c1) | 
+| :--: | :--: |
 
-> ### Basic Eyeblink & Talking
-> 1. Click **[Segmentation (SAM)]/ Edit prompt** button
->    
-> 2. Add new layer and rename
->
-> 3. Drag layer to **[Create Mesh Layer]/image**
-> 
-> 4. **[Create Mesh Layer]/ face_threshold, shape_threshold**, To control mesh threshold, recommend value: 0.6~0.7
->    
-> 5. **[Create Mesh Layer]/ scale_x, scale_y, extrude_x, extrude_y**, To control mesh threshold, recommend value: 1.2~1.4
->
-> 6. **[Modify Shape Key]/ rotate** Setting Reference, If Head tilted to the left, set a positive number angle 
-> 
-> |  <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/271264869-abf2a843-8ca5-44a6-9611-c334d55928d1.png" width="300"> |  <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/271264902-37658a8e-6f46-4c5b-bfd6-adec270df60b.png" width="300"> |  <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/271264910-0fae0c27-428d-4a5d-8296-6634c9717b95.png" width="300"> |  <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/271264920-4fea7882-cc51-4a5a-af9a-e66589810f92.png" width="300"> |
-> | --- | --- | --- | --- |
-> | 0 | 5 | -5 | -15 |
+Download: Save the image, and drag into Comfyui
+
+| ![ComfyUI_09609_](https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/caa98eec-4fb2-449d-9558-5d4a45e07580) | ![dog_480p15_high](https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/9fb701c9-f25c-408f-b96c-749773a53bd2) | 
+| :--: | :--: |
+
+Download: Save the image, and drag into Comfyui or [Dog Workflow](https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/Dog_workflow.json)
+
+# Best practices for image input
+
+### 1. Generate a new character image
+
+![image](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270609114-acea9933-359b-4398-8d2a-582bf02bef99.png)
+
+We need a character image with an open mouth and enable the tool to easily recognize facial features, so please add to the prompt:
+`looking at viewer, detailed face, open mouth, [smile], solo,eye-level angle`
+
+Download: [Character Gen Template](https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/SimpleCharacterGen.json)
+
+### 2. Make existing character image mouth open (Inpaint)
+
+![inpaint_workflow](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270589181-d11d840b-7ea6-4b47-bc26-a2af7c8c27a5.png)
+
+To maintain consistency with the base image, it is recommended to utilize a checkpoint model that aligns with its style.
+
+Download: [Mouth Open Inpaint Template](<https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/MouthOpen_(inpaint).json>)
+
+<details>
+<summary> Inpaint Demonstration </summary>
+
+<video src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/e3b77295-a1bf-4d96-9551-7cc423a4af73"/>
 
 </details>
 
-<details> 
-    
-<summary> Template01 - ControlNet Gen Guide </summary>
+### 3. Pose Constraints (ControlNet)
+
+![image](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270943267-c3cae113-2df4-45f2-a19c-885cbee75450.png)
 
 Place normal and openpose image with reference to images.
 
-![image](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270943267-c3cae113-2df4-45f2-a19c-885cbee75450.png)
-    
-</details>
+Download: [ControlNet Gen](https://github.com/avatechai/avatar-graph-comfyui/tree/main/workflow_templates/TemplateGen01)
 
 
-    
-# Image Preprocess Guide
+# Recommend Checkpoint Model List
 
-### 💡If you want to generate a new character image
-> you can download this Template and refer to the Guide!
-> <details> 
-> <summary> Character Gen Prompting Guide </summary>
->
-> # Character Gen Prompting Guide
->> **🎯Notice**
->> 
->> We need a character image with an open mouth and enable the tool to easily recognize facial features, so please add to the prompt:
->>
->> ```looking at viewer, detailed face, open mouth, [smile], solo,eye-level angle```
->  
->![image](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270609114-acea9933-359b-4398-8d2a-582bf02bef99.png)
-> 
-> ### Download: 📂[Character Gen Template](https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/SimpleCharacterGen.json)
-> Feel free to change any checkpoint model that suits your needs.
->  
-> </details>
+##### Anime Style SD1.5
 
-### 💡If you have a character image but it's not mouth open
-> you can download this Template and refer to the Guide!
-> <details> 
-> <summary> Mouth Open Guide (Inpaint) </summary>
->
-> # Mouth Open Guide (Inpaint)
-> To maintain consistency with the base image, it is recommended to utilize a checkpoint model that aligns with its style.
->
-> ![inpaint_workflow](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270589181-d11d840b-7ea6-4b47-bc26-a2af7c8c27a5.png)
->
-> ### Download: 📂[MouthOpen Template](https://github.com/avatechai/avatar-graph-comfyui/blob/main/workflow_templates/MouthOpen_(inpaint).json)
->
-> ### Inpaint Demonstration 
->
-> <video src="https://github.com/avatechai/avatar-graph-comfyui/assets/73209427/e3b77295-a1bf-4d96-9551-7cc423a4af73"/>
-> 
-> ### Recommend Checkpoint Model List 
-> 
-> ##### Anime Style SD1.5
->- https://civitai.com/models/35960/flat-2d-animerge
->- https://civitai.com/models/24149/mistoonanime
->- https://civitai.com/models/22364/kizuki-anime-hentai-checkpoint
->##### Realistic Style SD1.5
->- https://civitai.com/models/4201/realistic-vision-v51
->- https://civitai.com/models/49463/am-i-real
->- https://civitai.com/models/43331/majicmix-realistic
-> 
-> </details>
+- https://civitai.com/models/35960/flat-2d-animerge
+- https://civitai.com/models/24149/mistoonanime
+
+##### Realistic Style SD1.5
+
+- https://civitai.com/models/4201/realistic-vision-v51
+- https://civitai.com/models/49463/am-i-real
+- https://civitai.com/models/43331/majicmix-realistic
 
 # Custom Nodes
-Expand to see all the available nodes description
-Mesh Edit Nodes
-Shape Keys Nodes
-Avatar Output Nodes
+
+Expand to see all the available nodes description.
 
 <details>
-<summary> Image Segmentation Nodes </summary>
+<summary> All Custom Nodes </summary>
 
-## Image Segmentation Nodes
-| Name                         | Description  | Preview |
-| ---------------------------- | ------------ | ------- |
-| `Segmentation (SAM)`         | Integrative SAM node allowing you to directly select and create multiple image segment output. | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270576351-8aabeba8-5450-4d39-8203-e91f9ab47190.png" width="300"> |
+| Name                 | Description                                                                                    | Preview                                                                                                                                                              |
+| -------------------- | ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Segmentation (SAM)` | Integrative SAM node allowing you to directly select and create multiple image segment output. | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270576351-8aabeba8-5450-4d39-8203-e91f9ab47190.png" width="300"> |
+
+
+| Name                         | Description                                                                                     | Preview                                                                                                                                                              |
+| ---------------------------- | ----------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Create Mesh Layer`          | Create a mesh object from the input images (usually a segmented part of the entire image)       | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270576646-40740d25-9411-4cd3-a6c0-8b9008bca41c.png" width="300"> |
+| `Join Meshes`                | Combine multiple meshes into a single mesh object                                               | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270577004-ba7afbc5-9cd5-4f97-9614-f71133f5783e.png" width="300"> |
+| `Match Texture Aspect Ratio` | Since the mesh is created in 1:1 aspect ratio, a re-scale is needed at the end of the operation | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270602782-cb7155be-fb31-49f8-a24a-d001a1484ea7.png" width="300"> |
+| `Plane Texture Unwrap`       | Will perform mesh face fill and UV Cube project on the target plane mesh, scaled to bounds.     | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270603006-4b9c0cf5-0497-47bf-8e06-5a3370084c11.png" width="300"> |
+
+| Name                    | Description                                                                                        | Preview                                                                                                                                                              |
+| ----------------------- | -------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Mesh Modify Shape Key` | Given shape key name & target vertex_group, modify the vertex / all vertex’s transform             | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270577944-ab4f259c-89a7-4f51-bc54-fd179e252073.png" width="300"> |
+| `Create Shape Flow`     | Create runtime shape flow graph, allowing interactive inputs affecting shape keys value in runtime | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270592752-abfdd801-0387-4c5d-9c11-6c23337ff1dd.png" width="300"> |
+
+| Name                 | Description                                                                                           | Preview                                                                                                                                                              |
+| -------------------- | ----------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `Avatar Main Output` | The primary output of the .ava file. The embedded Avatar View will auto update with this node's output | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270592519-6a9a8bb4-05ec-4a2e-98bf-194b6af3a62a.png" width="300"> |
 
 </details>
 
-<details>
-<summary> Mesh Edit Nodes </summary>
-
-## Mesh Edit Nodes
-
-| Name                         | Description                                                                                           | Preview                                                                                                                        |
-| ---------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
-| `Create Mesh Layer`          | Create a mesh object from the input images (usually a segmented part of the entire image)             | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270576646-40740d25-9411-4cd3-a6c0-8b9008bca41c.png" width="300"> |
-| `Join Meshes`                | Combine multiple meshes into a single mesh object                                                     | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270577004-ba7afbc5-9cd5-4f97-9614-f71133f5783e.png" width="300"> |
-| `Match Texture Aspect Ratio` | Since the mesh is created in 1:1 aspect ratio, a re-scale is needed at the end of the operation       | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270602782-cb7155be-fb31-49f8-a24a-d001a1484ea7.png" width="300"> |
-| `Plane Texture Unwrap`       | Will perform mesh face fill and UV Cube project on the target plane mesh, scaled to bounds.           | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270603006-4b9c0cf5-0497-47bf-8e06-5a3370084c11.png" width="300"> |
-
-
-</details>
-
-<details>
-<summary> Shape Keys Nodes </summary>
-
-## Shape Keys Nodes
-| Name                         | Description                                                                                           | Preview                                                                                                                        |
-| ---------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
-| `Mesh Modify Shape Key`      | Given shape key name & target vertex_group, modify the vertex / all vertex’s transform                | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270577944-ab4f259c-89a7-4f51-bc54-fd179e252073.png" width="300"> |
-| `Create Shape Flow`          | Create runtime shape flow graph, allowing interactive inputs affecting shape keys value in runtime    | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270592752-abfdd801-0387-4c5d-9c11-6c23337ff1dd.png" width="300"> |
-
-</details>
-
-<details>
-<summary> Avatar Output Nodes </summary>
-
-## Avatar Output Nodes
-| Name                         | Description                                                                                           | Preview                                                                                                                        |
-| ---------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
-| `Avatar Main Output`         | The primary output of the .ava file. The embeded Avatar View will auto update with this node's output | <img src="https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270592519-6a9a8bb4-05ec-4a2e-98bf-194b6af3a62a.png" width="300"> |
-
-</details>
-
-
-
 # Shape Flow
+
 ![image](https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/public-download/github-readme/270618471-a834e535-4f87-4b77-81a6-435e3a67ca4a.png)
 
 # Installation
 
-Clone the repository to custom_nodes in your [ComfyUI](https://github.com/comfyanonymous/ComfyUI) directory:
+## Method 1 - Windows
+
+1. Download Python environment from [here](https://avatech-avatar-dev1.nyc3.digitaloceanspaces.com/comfyui/ComfyUI_3.10.7z)
+
+2. Unzip it to ComfyUI directory
+
+3. Run the `run_cpu_3.10.bat` or `run_nvidia_gpu_3.10.bat`
+
+4. Install avatar-graph-comfyui from [ComfyUI Manager](https://github.com/ltdrdata/ComfyUI-Manager)
+
+## Method 2 - macOS/Linux
+
+Make sure your Python environment is `3.10.x` as required by the [bpy](https://pypi.org/project/bpy/) package. Then go to the [ComfyUI](https://github.com/comfyanonymous/ComfyUI) directory and run:
+
+> Suggest using conda for your comfyui python environment
+> 
+> `conda create --name comfyui python=3.10`
+> 
+> `conda activate comfyui`
+> 
+> `pip install -r requirements.txt`
 
 1. `cd custom_nodes`
 
 2. `git clone https://github.com/avatechgg/avatar-graph-comfyui.git`
 
-3. Install deps `cd avatar-graph-comfyui && python -m pip install -r requirements.txt`
+3. `cd avatar-graph-comfyui && python -m pip install -r requirements.txt`
 
-4. Restart comfyui
-
-5. Run comfyui with enable-cors-header `python main.py --enable-cors-header` or (mac)`python main.py --force-fp16 --enable-cors-header`
+4. Restart ComfyUI with enable-cors-header `python main.py --enable-cors-header` or (for mac) `python main.py --force-fp16 --enable-cors-header`
 
 # Development
 
-<details>
-<summary> If you are interested in contributing expand to see development details </summary>
-
+If you are interested in contributing
 
 For comfyui frontend extension, frontend js located at `avatar-graph-comfyui/js`
 
@@ -228,9 +193,21 @@ For each changes, simply refresh the comfyui page to see the changes.
     ]
 }
 ```
-
 </details>
 
-</details>
+## Update blender node types
 
+To update blender operations input and output types (stored in `blender/input_types.txt`), run:
 
+```bash
+python generate_blender_types.py
+```
+
+# FAQ
+
+## What is `--enable-cors-header` used for?
+It is used to enable communication between ComfyUI and our editor (https://editor.avatech.ai), which is in charge of animating static characters. The only messages exchanged between them are the character data like the meshes of eyes and mouth, and the JSON format of our editor graph.
+
+When you execute the ComfyUI graph, it sends the character data and the JSON graph to our editor for animating. When you modify and save the graph in our editor, it sends the modified graph back to ComfyUI. To validate it, you can open the `js/index.js`, and log the message in `window.addEventListener("message", ...)` and `postMessage(message)`.
+
+You can also run ComfyUI *without* the `--enable-cors-header`: execute the ComfyUI workflow, then download the .GLB or .GLTF format by right clicking the Avatar Main Output node and Save File option. Yet, this will disable the real-time character preview in the top-right corner of ComfyUI. Feel free to view it in other software like Blender.

__init__.py

@@ -43,7 +43,29 @@ def append_to_sys_path(path):
     if path not in sys.path:
         sys.path.append(path)
 
-folder_paths.folder_names_and_paths["sams"] = ([os.path.join(folder_paths.models_dir, "sams")], folder_paths.supported_pt_extensions)
+
+folder_paths.folder_names_and_paths["sams"] = (
+    [os.path.join(folder_paths.models_dir, "sams")],
+    folder_paths.supported_pt_extensions,
+)
+
+
+def download_model(url, save_path):
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+    file_size = int(response.headers.get("Content-Length", 0))
+    chunk_size = 1024
+    num_bars = int(file_size / chunk_size)
+
+    with open(save_path, "wb") as f:
+        for chunk in tqdm(
+            response.iter_content(chunk_size=chunk_size),
+            total=num_bars,
+            unit="KB",
+            desc=url.split("/")[-1],
+        ):
+            f.write(chunk)
+
 
 def download_sam_model():
     model_dir = get_folder_paths("sams")[0]
@@ -53,27 +75,35 @@ def download_sam_model():
     add_model_folder_path("sams", model_dir)
 
     files = get_filename_list("sams")
-    if len(files) == 0:
+    if "sam_vit_h_4b8939.pth" not in files:
         print("Downloading sam model...")
         url = "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth"
-        response = requests.get(url, stream=True)
-        response.raise_for_status()
-        file_size = int(response.headers.get("Content-Length", 0))
-        chunk_size = 1024
-        num_bars = int(file_size / chunk_size)
-
-        with open(f"{model_dir}/sam_vit_h_4b8939.pth", "wb") as f:
-            for chunk in tqdm(
-                response.iter_content(chunk_size=chunk_size),
-                total=num_bars,
-                unit="KB",
-                desc=url.split("/")[-1],
-            ):
-                f.write(chunk)
+        download_model(url, f"{model_dir}/sam_vit_h_4b8939.pth")
 
 
 download_sam_model()
 
+
+def download_face_and_pose_landmarker():
+    model_dir = os.path.join(ag_path, "mediapipe_models")
+    if not os.path.isdir(model_dir):
+        os.makedirs(model_dir)
+
+    model_path = os.path.join(model_dir, "face_landmarker.task")
+    if not os.path.isfile(model_path):
+        print("Downloading face landmarker model...")
+        url = "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task"
+        download_model(url, model_path)
+
+    model_path = os.path.join(model_dir, "pose_landmarker_full.task")
+    if not os.path.isfile(model_path):
+        print("Downloading pose landmarker model...")
+        url = "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/latest/pose_landmarker_full.task"
+        download_model(url, model_path)
+
+
+download_face_and_pose_landmarker()
+
 paths = ["blender", "sam"]
 files = []
 

blender/avatar_main_output.py

@@ -1,6 +1,6 @@
 import platform
 import blender_node
-from mesh_utils import assign_texture, open_in_blender as open_blender, export_gltf
+from mesh_utils import upload_avatar_file, open_in_blender as open_blender, export_gltf
 import folder_paths
 
 global_blender_path = ''
@@ -47,14 +47,16 @@ class AvatarMainOutput(blender_node.ObjectOps):
         }),
         "model_type": (["AVA","GLB", "GLTF_EMBEDDED"],),
         "write_mode": (["Overwrite", "Increment"],),
-
+        "upload_to_cloud": ("BOOLEAN", {
+            "default": False
+        }),
         "SHAPE_FLOW": ("SHAPE_FLOW",),
     }
 
     OUTPUT_NODE = True
     RETURN_TYPES = ()
 
-    def blender_process(self, bpy, BPY_OBJ=None, BPY_OBJS=None, open_in_blender=False, auto_save=False, blender_path_override='', filename='', model_type='', write_mode='', SHAPE_FLOW=''):
+    def blender_process(self, bpy, BPY_OBJ=None, BPY_OBJS=None, open_in_blender=False, auto_save=False, blender_path_override='', filename='', model_type='', write_mode='', upload_to_cloud=False, SHAPE_FLOW=''):
         if open_in_blender:
             p = blender_path_override if blender_path_override else global_blender_path
             output_file = self.output_dir + '/tmp.blend'
@@ -72,4 +74,23 @@ class AvatarMainOutput(blender_node.ObjectOps):
         import global_bpy
         global_bpy.set_should_reset_scene(True)
 
-        return {"ui": {"gltfFilename": {filepath.replace(f"{self.output_dir}/", "")}, "SHAPE_FLOW": {SHAPE_FLOW}, "auto_save": {'true' if auto_save else 'false'},}}
+        outputs = {
+            "gltfFilename": [filepath.replace(f"{self.output_dir}/", "")],
+            "files": [{
+                "filename": filepath.replace(f"{self.output_dir}/", ""),
+                "content_type": "model/gltf+json",
+            },],
+            "SHAPE_FLOW": {SHAPE_FLOW}, 
+            "auto_save": {'true' if auto_save else 'false'}, 
+        }
+        if upload_to_cloud:
+            avatarId = upload_avatar_file(outputs)
+            return {
+                "ui": {
+                    **outputs,
+                    "avatarId": [avatarId],
+                },
+            }
+        return {
+            "ui": outputs
+        }

blender/b_vector3d.py

@@ -5,17 +5,17 @@ class VECTOR3D:
             "required": {
                 "x": ("FLOAT", {
                     "default": 0,
-                    "step": 0.1,
+                    "step": 0.01,
                     "display": "number"
                 }),
                 "y": ("FLOAT", {
                     "default": 0,
-                    "step": 0.1,
+                    "step": 0.01,
                     "display": "number"
                 }),
                 "z": ("FLOAT", {
                     "default": 0,
-                    "step": 0.1,
+                    "step": 0.01,
                     "display": "number"
                 }),
             },

blender/b_vector4d.py

@@ -5,17 +5,17 @@ class VECTOR4D:
             "required": {
                 "x": ("FLOAT", {
                     "default": 0,
-                    "step": 0.1,
+                    "step":  0.01,
                     "display": "number"
                 }),
                 "y": ("FLOAT", {
                     "default": 0,
-                    "step": 0.1,
+                    "step":  0.01,
                     "display": "number"
                 }),
                 "z": ("FLOAT", {
                     "default": 0,
-                    "step": 0.1,
+                    "step":  0.01,
                     "display": "number"
                 }),
                 "u": ("FLOAT", {

blender/blender_node.py

@@ -1,6 +1,7 @@
 import inspect
 import re
 import json
+import os
 
 BPY_OBJS = "BPY_OBJS"
 BPY_OBJ = "BPY_OBJ"
@@ -10,12 +11,13 @@ BPY_OBJS_TYPE = {
 }
 
 node_input_types = {}
-with open("custom_nodes/avatar-graph-comfyui/blender/input_types.txt") as f:
+with open(f"{os.path.dirname(__file__)}/input_types.txt") as f:
     input_types = f.readlines()
     for input_type in input_types:
         node_cls, node_types = input_type.split("|")
         node_input_types[node_cls] = json.loads(node_types)
 
+type_generation = os.getenv('TYPE_GENERATION', 0)
 
 class ObjectOps:
     @classmethod
@@ -47,22 +49,29 @@ class ObjectOps:
 
     @classmethod
     def INPUT_TYPES(cls):
-        return node_input_types[cls.__name__]
+        if type_generation:
+            import global_bpy
 
-        # import global_bpy
-        # bpy = global_bpy.get_bpy()
-        # result = {
-        #     "required": {},
-        #     "optional": {
-        #         **cls.get_base_input_types(bpy),
-        #         **cls.get_extra_input_types(bpy)
-        #     }
-        # }
+            bpy = global_bpy.get_bpy()
+            result = {
+                "required": {},
+                "optional": {
+                    **cls.get_base_input_types(bpy),
+                    **cls.get_extra_input_types(bpy),
+                },
+            }
 
-        # with open("input_types.txt", "a") as f:
-        #     f.write(cls.__name__ + "|" + json.dumps(result) + "\n")
-
-        # return result
+            return result
+        elif cls.__name__ in node_input_types:
+            return node_input_types[cls.__name__]
+        else:
+            return {
+                "required": {},
+                "optional": {
+                    **cls.get_base_input_types(None),
+                    **cls.get_extra_input_types(None),
+                },
+            }
 
     @classmethod
     def NODE_CLASS_MAPPINGS(cls):
@@ -91,14 +100,14 @@ class ObjectOps:
         import global_bpy
         bpy = global_bpy.get_bpy()
 
-        if props.get("BPY_OBJ") != None:
+        if props.get("BPY_OBJ") is not None:
             bpy.context.view_layer.objects.active = props["BPY_OBJ"]
 
         results = self.blender_process(bpy, **props)
 
         if results is None:
             # print(results)
-            if props.get("BPY_OBJ") != None:
+            if props.get("BPY_OBJ") is not None:
                 return (props["BPY_OBJ"], )
             else:
                 return (bpy.context.view_layer.objects.active, )
@@ -251,7 +260,8 @@ def create_primitive_shape_class(cls, path, name=None, name_prefix=''):
 
 
 def assign_and_return(BPY_OBJ, name, value):
-    BPY_OBJ[name] = value
+    setattr(BPY_OBJ, name, value)
+    # BPY_OBJ[name] = value
     # print(BPY_OBJ,name, BPY_OBJ[name])
     return None
 

blender/image_alpha_mask_merge.py

@@ -0,0 +1,32 @@
+import folder_paths
+import os
+from PIL import Image, ImageOps
+from PIL.PngImagePlugin import PngInfo
+import numpy as np
+import torch
+import json
+
+class ImageAlphaMaskMerge:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {"required":
+                    {"image": ("IMAGE",) ,
+                    "mask": ("MASK",) },
+                }
+
+    CATEGORY = "image"
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "load_image"
+    def load_image(self, image, mask):
+        if image.shape[1] == mask.shape[0] and image.shape[2] == mask.shape[1]:
+            image = torch.cat((image, 1 - mask.unsqueeze(0).unsqueeze(3)), dim=3)
+        return (image, )
+
+NODE_CLASS_MAPPINGS = {
+     "Image Alpha Mask Merge": ImageAlphaMaskMerge,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "Image Alpha Mask Merge": "Image Alpha Mask Merge"
+}

blender/input_types.txt

@@ -1,17 +1,3 @@
-Object_VertexGroupNewWithName|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "name": ["STRING", {"multiline": false, "default": "Group"}], "assign_selected": ["BOOLEAN", {"default": true}]}}
-EditOps|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"]}}
-ContextSet_TransformPivotPoint|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "pivot": [["BOUNDING_BOX_CENTER", "CURSOR", "INDIVIDUAL_ORIGINS", "MEDIAN_POINT", "ACTIVE_ELEMENT"]]}}
-Object_AddShapeKeys|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "shape_keys": ["STRING", {"default": "key1,key2", "multiline": true}], "from_mix": ["BOOLEAN", {"default": false}]}}
-Object_MeshFromTexture|{"required": {}, "optional": {"image": ["IMAGE"], "seed": ["INT", {"default": 0, "min": 0, "max": 18446744073709551615}]}}
-Object_MatchTextureAspectRatio|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "image": ["IMAGE"], "scale": ["FLOAT", {"default": 0.001, "display": "number", "step": 0.001}]}}
-AssignVertexGroupOps|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "name": ["STRING", {"multiline": false, "default": "Group"}]}}
-Object_VertexGroupNewWithName|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "name": ["STRING", {"multiline": false, "default": "Group"}], "assign_selected": ["BOOLEAN", {"default": true}]}}
-Object_CreateMeshLayer|{"required": {}, "optional": {"image": ["IMAGE"], "convex_hull": ["BOOLEAN", {"default": true}], "shape_threshold": ["FLOAT", {"display": "number", "default": 0.7}], "mesh_layer_name": ["STRING", {"default": "mesh_layer"}], "scale_x": ["FLOAT", {"display": "number", "default": 1}], "scale_y": ["FLOAT", {"display": "number", "default": 1}], "extrude_x": ["FLOAT", {"display": "number", "default": 0}], "extrude_y": ["FLOAT", {"display": "number", "default": 0}], "seed": ["INT", {"default": 0, "min": 0, "max": 18446744073709551615}]}}
-GetImageWidthHeight|{"required": {}, "optional": {"image": ["IMAGE"], "scale": ["FLOAT", {"default": 1.0}]}}
-GetFirstObjOps|{"required": {}, "optional": {"BPY_OBJS": ["BPY_OBJS"]}}
-Object_AssignTexture|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "texture": ["IMAGE"], "texture_name": ["STRING", {"multiline": false, "default": "my_image"}]}}
-Mesh_JoinMesh|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "BPY_OBJ2": ["BPY_OBJ"]}}
-Mesh_ModifyShapeKey|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "shape_key_name": ["STRING", {"multiline": false, "default": "EyeBlinkLeft"}], "target_vertex_group": ["STRING", {"multiline": false, "default": ""}], "scale_x": ["FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}], "scale_y": ["FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}], "offset_x": ["FLOAT", {"default": 0.0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}], "offset_y": ["FLOAT", {"default": 0.0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}], "rotate": ["FLOAT", {"default": 0, "min": -360, "max": 360.0, "step": 0.01, "display": "number"}], "origin_offset_x": ["FLOAT", {"default": 0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}], "origin_offset_y": ["FLOAT", {"default": 0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}], "transform_radius": ["FLOAT", {"default": 1.0, "min": 0, "max": 1, "step": 0.01, "display": "number"}], "falloff": ["FLOAT", {"default": 0, "min": -10.0, "max": 10.0, "step": 0.01, "display": "number"}]}}
 Mesh_AttributeSet|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "value_float": ["FLOAT", {"min": -3.4028234663852886e+38, "max": 3.4028234663852886e+38, "default": 0.0}], "value_float_vector_2d": ["B_VECTOR2", {}], "value_float_vector_3d": ["B_VECTOR3", {}], "value_int": ["INT", {"min": -2147483648, "max": 2147483647, "default": 0}], "value_int_vector_2d": ["B_VECTOR2", {}], "value_color": ["B_VECTOR4", {}], "value_bool": ["BOOLEAN", {"default": false}]}}
 Mesh_AverageNormals|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "average_type": [["CUSTOM_NORMAL", "FACE_AREA", "CORNER_ANGLE"]], "weight": ["INT", {"min": 1, "max": 100, "default": 50}], "threshold": ["FLOAT", {"min": 0.0, "max": 10.0, "default": 0.009999999776482582}]}}
 Mesh_BeautifyFill|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "angle_limit": ["FLOAT", {"min": 0.0, "max": 3.1415927410125732, "default": 3.1415927410125732}]}}
@@ -607,8 +593,5 @@ UV_SnapCursor|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "target": [[
 UV_SnapSelected|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "target": [["PIXELS", "CURSOR", "CURSOR_OFFSET", "ADJACENT_UNSELECTED"]]}}
 UV_SphereProject|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "direction": [["VIEW_ON_EQUATOR", "VIEW_ON_POLES", "ALIGN_TO_OBJECT"]], "align": [["POLAR_ZX", "POLAR_ZY"]], "pole": [["PINCH", "FAN"]], "seam": ["BOOLEAN", {"default": false}], "correct_aspect": ["BOOLEAN", {"default": true}], "clip_to_bounds": ["BOOLEAN", {"default": false}], "scale_to_bounds": ["BOOLEAN", {"default": false}]}}
 UV_Stitch|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "use_limit": ["BOOLEAN", {"default": false}], "snap_islands": ["BOOLEAN", {"default": true}], "limit": ["FLOAT", {"min": 0.0, "max": 3.4028234663852886e+38, "default": 0.009999999776482582}], "static_island": ["INT", {"min": 0, "max": 2147483647, "default": 0}], "active_object_index": ["INT", {"min": 0, "max": 2147483647, "default": 0}], "midpoint_snap": ["BOOLEAN", {"default": false}], "clear_seams": ["BOOLEAN", {"default": true}], "mode": [["VERTEX", "EDGE"]], "stored_mode": [["VERTEX", "EDGE"]], "objects_selection_count": ["B_VECTOR6", {}]}}
+UV_Unwrap|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "method": [["ANGLE_BASED", "CONFORMAL"]], "fill_holes": ["BOOLEAN", {"default": true}], "correct_aspect": ["BOOLEAN", {"default": true}], "use_subsurf_data": ["BOOLEAN", {"default": false}], "margin_method": [["SCALED", "ADD", "FRACTION"]], "margin": ["FLOAT", {"min": 0.0, "max": 1.0, "default": 0.0010000000474974513}]}}
 UV_Weld|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"]}}
-ToGroupOps|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"]}}
-AvatarMainOutput|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "BPY_OBJS": ["BPY_OBJS"], "open_in_blender": ["BOOLEAN", {"default": false}], "auto_save": ["BOOLEAN", {"default": false}], "blender_path_override": ["STRING", {"multiline": false, "default": ""}], "filename": ["STRING", {"multiline": false, "default": "out"}], "model_type": [["AVA", "GLB", "GLTF_EMBEDDED"]], "write_mode": [["Overwrite", "Increment"]], "SHAPE_FLOW": ["SHAPE_FLOW"]}}
-GroupOps|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "BPY_OBJ2": ["BPY_OBJ"]}}
-Object_PlaneTextureUnwrap|{"required": {}, "optional": {"BPY_OBJ": ["BPY_OBJ"], "image": ["IMAGE"], "scale": ["FLOAT", {"default": 1, "display": "number", "step": 0.01}], "texture_name": ["STRING", {"default": "Texture"}]}}

blender/load_image_with_alpha.py

@@ -0,0 +1,55 @@
+import folder_paths
+import os
+from PIL import Image, ImageOps
+from PIL.PngImagePlugin import PngInfo
+import numpy as np
+import torch
+import json
+
+class LoadImageWithAlpha:
+    @classmethod
+    def INPUT_TYPES(s):
+        input_dir = folder_paths.get_input_directory()
+        files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
+        return {"required":
+                    {"image": (sorted(files), {"image_upload": True})},
+                }
+
+    CATEGORY = "image"
+
+    RETURN_TYPES = ("IMAGE",)
+    FUNCTION = "load_image"
+    def load_image(self, image):
+        image_path = folder_paths.get_annotated_filepath(image)
+        i = Image.open(image_path)
+        i = ImageOps.exif_transpose(i)
+        image = i.convert("RGBA")
+        image = np.array(image).astype(np.float32) / 255.0
+        image = torch.from_numpy(image)[None,]
+
+        print(image.shape)
+
+        return (image, )
+
+    @classmethod
+    def IS_CHANGED(s, image):
+        image_path = folder_paths.get_annotated_filepath(image)
+        m = hashlib.sha256()
+        with open(image_path, 'rb') as f:
+            m.update(f.read())
+        return m.digest().hex()
+
+    @classmethod
+    def VALIDATE_INPUTS(s, image):
+        if not folder_paths.exists_annotated_filepath(image):
+            return "Invalid image file: {}".format(image)
+
+        return True
+
+NODE_CLASS_MAPPINGS = {
+     "LoadImageWithAlpha": LoadImageWithAlpha,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "LoadImageWithAlpha": "LoadImageWithAlpha"
+}

blender/mesh_utils.py

@@ -1,6 +1,9 @@
 import atexit
 import subprocess
 import os
+import folder_paths
+import requests
+import json
 
 def genreate_mesh_from_texture(bpy, image):
     import torch
@@ -14,6 +17,9 @@ def genreate_mesh_from_texture(bpy, image):
     contours, _ = cv2.findContours(
         gray, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
 
+    if len(contours) == 0:
+        raise Exception("No contours found. Please ensure that the image has the correct segments (e.g. when you click on the mouth, it should display a proper blue area over the mouth region).")
+
     # Get the largest contour
     areas = [cv2.contourArea(contour) for contour in contours]
 
@@ -86,7 +92,7 @@ def assign_texture(bpy, BPY_OBJ, texture, texture_name):
 
     # Create an image with the required dimensions
     img = bpy.data.images.new(
-        texture_name, width=texture.shape[1], height=texture.shape[0])
+        texture_name, width=texture.shape[1], height=texture.shape[0], alpha = True)
 
     # If there is no alpha channel, append one full of 1's
     if texture.shape[2] == 3:
@@ -118,6 +124,7 @@ def assign_texture(bpy, BPY_OBJ, texture, texture_name):
     # Create a material
     mat = bpy.data.materials.new("MaterialName")
     mat.use_nodes = True
+    mat.blend_method = 'BLEND'
     nodes = mat.node_tree.nodes
     for node in nodes:
         nodes.remove(node)
@@ -138,6 +145,8 @@ def assign_texture(bpy, BPY_OBJ, texture, texture_name):
               texture_node.outputs['Color'])
     links.new(output_node.inputs['Surface'], bsdf_node.outputs['BSDF'])
 
+    links.new(bsdf_node.inputs['Alpha'], texture_node.outputs['Alpha'])
+
     # Assign the material to the active object
     if obj.data.materials:
         obj.data.materials[0] = mat
@@ -236,7 +245,42 @@ def export_gltf(output_dir, bpy_objects, filename, model_type, write_mode, metad
         # print(filepath)
         if filepath.endswith('.ava.glb'):
             new_filepath = filepath.replace('.ava.glb', '.ava')
-            os.rename(filepath, new_filepath)
+            os.replace(filepath, new_filepath)
             filepath = new_filepath
 
-    return filepath
+    return filepath
+
+def get_avatar_file(output):
+    avatar_filename = output["gltfFilename"][0]
+    with open(
+        f"{folder_paths.get_output_directory()}/{avatar_filename}", "rb"
+    ) as f:
+        return f.read()
+
+def upload_avatar_file(output):
+    file = get_avatar_file(output)
+    response = requests.get("https://labs.avatech.ai/api/share")
+    labData = response.json()
+    modelId = labData["modelId"]
+
+    # upload model
+    headers = {
+        "x-amz-acl": "public-read",
+        "Content-Type": "model/gltf-binary",
+        "Content-Length": str(len(file)),
+    }
+    requests.put(labData["url"], headers=headers, data=file)
+
+    # send notification
+    webhook_url = os.getenv("DISCORD_WEBHOOK_URL")
+    data = {
+        "username": "Avabot",
+        "avatar_url": "https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/avatechai.png",
+        "content": "[API Call] New register!",
+    }
+    headers = {
+        "Content-Type": "application/json",
+    }
+    response = requests.post(webhook_url, headers=headers, data=json.dumps(data))
+
+    return modelId

blender/ops_create_mesh_layer_advanced.py

@@ -0,0 +1,62 @@
+import blender_node
+from mesh_utils import genreate_mesh_from_texture
+
+class Object_CreateMeshLayer_Advanced(blender_node.ObjectOps):
+
+    BASE_INPUT_TYPES = {}
+    
+    CUSTOM_NAME = "Create Mesh Layer (Advanced)"
+
+    EXTRA_INPUT_TYPES = {
+        "image": ("IMAGE",),
+        "convex_hull": ("BOOLEAN", {"default": True}),
+        # "face_threshold": ("FLOAT", {"display": "number", "default": 0.7}),
+        "shape_threshold": ("FLOAT", {"display": "number", "default": 0.7}),
+        "mesh_layer_name": ("STRING", {"default": "mesh_layer"}),
+        "scale_x": ("FLOAT", {"display": "number", "default": 1}),
+        "scale_y": ("FLOAT", {"display": "number", "default": 1}),
+        "extrude_x": ("FLOAT", {"display": "number", "default": 0}),
+        "extrude_y": ("FLOAT", {"display": "number", "default": 0}),
+        "inner_translate_x": ("FLOAT", {"display": "number", "default": 0, "step": 0.01}),
+        "inner_translate_y": ("FLOAT", {"display": "number", "default": 0, "step": 0.01}),
+        "outer_translate_x": ("FLOAT", {"display": "number", "default": 0, "step": 0.01}),
+        "outer_translate_y": ("FLOAT", {"display": "number", "default": 0, "step": 0.01}),
+        "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+    }
+
+    RETURN_TYPES = ("BPY_OBJ", "IMAGE")
+
+    def blender_process(self, bpy, image, convex_hull, shape_threshold, mesh_layer_name, scale_x,scale_y , extrude_x, extrude_y, inner_translate_x, inner_translate_y, outer_translate_x, outer_translate_y, seed):
+        image, BPY_OBJ = genreate_mesh_from_texture(bpy, image)
+
+        bpy.context.view_layer.objects.active = BPY_OBJ
+
+        self.edit_mode(bpy)
+
+        if convex_hull:
+            bpy.ops.mesh.convex_hull(
+                delete_unused=True, use_existing_faces=True,
+                shape_threshold=shape_threshold,
+                # face_threshold=face_threshold
+                face_threshold=0.7
+            )
+
+        bpy.ops.mesh.delete(type='EDGE_FACE')
+        bpy.ops.mesh.select_all(action='SELECT')
+        bpy.ops.mesh.edge_face_add()
+
+        bpy.ops.transform.resize(value=(scale_x, scale_y, 1))
+        bpy.ops.transform.translate(value=(inner_translate_x, inner_translate_y, 0))
+
+        bpy.context.object.vertex_groups.new(name=mesh_layer_name)
+        bpy.ops.object.vertex_group_assign()
+
+        if extrude_x != 0 or extrude_y != 0:
+            bpy.ops.mesh.extrude_region_move()
+            bpy.ops.object.vertex_group_remove_from()
+            bpy.ops.transform.resize(value=(extrude_x, extrude_y, 0))
+            bpy.ops.transform.translate(value=(outer_translate_x, outer_translate_y, 0))
+            bpy.ops.mesh.delete(type='ONLY_FACE')
+
+        self.object_mode(bpy)
+        return (BPY_OBJ, image)

blender/ops_group.py

@@ -8,5 +8,6 @@ class GroupOps(blender_node.ObjectOps):
     RETURN_TYPES = (blender_node.BPY_OBJS,)
 
     def blender_process(self, bpy, BPY_OBJ, BPY_OBJ2, **props):
-        return ([BPY_OBJ, BPY_OBJ2],)
+        prop_values = props.values()
+        return ([BPY_OBJ, BPY_OBJ2, *prop_values],)
 

blender/ops_join.py

@@ -11,8 +11,10 @@ class Mesh_JoinMesh(blender_node.ObjectOps):
     def blender_process(self, bpy, BPY_OBJ, **props):
         prop_values = props.values()        
         for obj in list(prop_values) + [BPY_OBJ]:
-            obj.select_set(True)
-        bpy.context.view_layer.objects.active = BPY_OBJ
+            if obj is not None:
+                obj.select_set(True)
+        if bpy.context.view_layer.objects is not None:
+            bpy.context.view_layer.objects.active = BPY_OBJ
         bpy.ops.object.join()
 
         return (BPY_OBJ,)

blender/ops_modify_uv.py

@@ -0,0 +1,24 @@
+import blender_node
+from mesh_utils import genreate_mesh_from_texture, assign_texture
+
+class Object_UV_Modifier(blender_node.EditOps):
+
+    EXTRA_INPUT_TYPES = {
+        "scale": ('FLOAT', {'default': 1, "display": "number", "step": 0.01}),
+        "texture_name": ('STRING', {'default': 'Texture', })
+    }
+
+    CUSTOM_NAME = "UV Modifier"
+
+    def blender_process(self, bpy, BPY_OBJ, scale, texture_name):
+        import bmesh
+
+        bm = bmesh.from_edit_mesh(BPY_OBJ.data)
+
+        uv_layer = bm.loops.layers.uv.verify()
+        for f in bm.faces:
+            # move all of the UVs in this face up one UDIM tile
+            for l in f.loops:
+                l[uv_layer].uv = (l[uv_layer].uv[0], 0.998 if l[uv_layer].uv[1] == 1 else l[uv_layer].uv[1])
+
+        bmesh.update_edit_mesh(BPY_OBJ.data)

blender/save_image_exclude.py

@@ -0,0 +1,82 @@
+import folder_paths
+import os
+from PIL import Image, ImageOps
+from PIL.PngImagePlugin import PngInfo
+import numpy as np
+import torch
+import json
+
+class SaveImageWithWorkflow:
+    def __init__(self):
+        self.output_dir = folder_paths.get_output_directory()
+        self.type = "output"
+        self.prefix_append = ""
+
+    @classmethod
+    def INPUT_TYPES(s):
+        input_dir = folder_paths.get_input_directory()
+        files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f))]
+        return {"required": 
+                   {"image": (sorted(files), {"image_upload": True}),
+                    "filename_prefix": ("STRING", {"default": "ComfyUI"})},
+               "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
+               }
+
+    RETURN_TYPES = ()
+    FUNCTION = "save_images"
+
+    OUTPUT_NODE = True
+
+    CATEGORY = "image"
+
+    def save_images(self, image, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):
+        image_path = folder_paths.get_annotated_filepath(image)
+        i = Image.open(image_path)
+        i = ImageOps.exif_transpose(i)
+        image = i.convert("RGBA")
+        image = np.array(image).astype(np.float32) / 255.0
+        image = torch.from_numpy(image)[None,]
+        images=(image)
+        
+        filename_prefix += self.prefix_append
+        full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0])
+        results = list()
+        for image in images:
+            i = 255. * image.cpu().numpy()
+            img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
+            metadata = None
+            metadata = PngInfo()
+            if prompt is not None:
+                # prompt = json.loads(prompt)
+                prompt = {k: v for k, v in prompt.items() if v['class_type'] != 'Save Image With Workflow'}
+                metadata.add_text("prompt", json.dumps(prompt))
+
+            print(extra_pnginfo)
+
+            # if prompt is not None:
+            #     metadata.add_text("prompt", json.dumps(prompt))
+            if extra_pnginfo is not None:
+                for x in extra_pnginfo:
+                    if (x == 'workflow'):
+                        extra_pnginfo[x]["nodes"] = [node for node in extra_pnginfo[x]["nodes"] if node['type'] != 'Save Image With Workflow']
+
+                    metadata.add_text(x, json.dumps(extra_pnginfo[x]))
+
+            file = f"{filename}_{counter:05}_.png"
+            img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=4)
+            results.append({
+                "filename": file,
+                "subfolder": subfolder,
+                "type": self.type
+            })
+            counter += 1
+
+        return { "ui": { "images": results } }
+
+NODE_CLASS_MAPPINGS = {
+     "Save Image With Workflow": SaveImageWithWorkflow,
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "Save Image With Workflow": "Save Image With Workflow"
+}

generate_blender_types.py

@@ -0,0 +1,17 @@
+import os
+import sys
+
+os.environ["TYPE_GENERATION"] = "1"
+
+current_dir = os.path.dirname(__file__)
+blender_dir = os.path.join(current_dir, "blender")
+sys.path.append(blender_dir)
+
+import json
+from blender.ops_mesh import BLENDER_NODES
+
+
+with open(f"{blender_dir}/input_types.txt", "w") as f:
+    for node in BLENDER_NODES:
+        results = node.INPUT_TYPES()
+        f.write(node.__name__ + "|" + json.dumps(results) + "\n")

js/Alert.js

@@ -3,8 +3,25 @@ import { van } from "./van.js";
 const { div, span } = van.tags;
 
 export function Alert() {
+  const color = van.state("bg-orange-100 text-orange-700 border-orange-500");
+
   van.derive(() => {
     if (alertDialog.val.time > 0) {
+      switch (alertDialog.val.type) {
+        case "error":
+          color.val = "bg-red-100 text-red-700 border-red-500";
+          break;
+        case "success":
+          color.val = "bg-green-100 text-green-700 border-green-500";
+          break;
+        case "info":
+          color.val = "bg-blue-100 text-blue-700 border-blue-500";
+          break;
+        case "warning":
+        default:
+          color.val = "bg-orange-100 text-orange-700 border-orange-500";
+          break;
+      }
       setTimeout(() => {
         alertDialog.val = { text: "", time: 0 };
       }, alertDialog.val.time);
@@ -14,13 +31,12 @@ export function Alert() {
   return div(
     {
       class: () =>
-        "absolute bottom-8 flex justify-center w-full " +
+        "absolute z-[100] bottom-8 flex justify-center w-full " +
         (alertDialog.val.text ? "" : "hidden"),
     },
     div(
       {
-        class:
-          "bg-orange-100 border-t-4 border-orange-500 rounded-sm text-orange-700 p-2",
+        class: () => `${color.val} border-t-4 rounded-sm p-2`,
       },
       () => span(alertDialog.val.text)
     )

js/AppHeader.js

@@ -0,0 +1,28 @@
+import { van } from "./van.js";
+const { div, span } = van.tags;
+
+export function AppHeader() {
+  return div(
+    {
+      class: () => "absolute flex justify-between top-0 w-full text-white p-4",
+    },
+    div(
+      {},
+      span(
+        {
+          class:
+            "block bg-gradient-to-b from-gray-500 to-white text-transparent bg-clip-text text-2xl",
+        },
+        "Avatech v1"
+      ),
+      span(
+        {
+          class:
+            "bg-gradient-to-b from-gray-500 to-white text-transparent bg-clip-text text-lg",
+        },
+        "Get your DALLE3 AI Personal Clone"
+      )
+    ),
+    span({ class: "text-gray-300" }, "Twitter")
+  );
+}

js/AvatarPreview.js

@@ -1,25 +1,774 @@
 import { van } from "./van.js";
-const { button, iframe, div, img } = van.tags;
-import { showEditor, previewUrl } from "./state.js";
+import {
+  imageUrl,
+  showPreview,
+  previewUrl,
+  showEditor,
+  previewImg,
+  previewImgLoading,
+  alertDialog,
+  isGenerateFlow,
+  enableAutoSegment
+} from "./state.js";
+const { button, iframe, div, img, input, label, span, textarea, ul, li } =
+  van.tags;
+import { app } from "./app.js";
+import { uploadPreview } from "./index.js";
+import { api } from "./api.js";
+import { segmented, uploadSegments } from "./LayerEditor.js";
+import { initModel } from "./onnx.js";
+// import { uploadSegments } from "./LayerEditor.js";
+
+const workflowList = [
+  "idle_avatar_(trigger)",
+  "Auto_segment_workflow",
+  "BronyaZaychik_(ChinaDress)",
+  "BronyaZaychik_(Default_Silverwing)",
+  "BronyaZaychik_(Non-official_office_ladysuit)",
+  "BronyaZaychik_(Official_office_ladysuit)",
+  "BronyaZaychikLora_withhand",
+  "SilverWolf_(Default)",
+  "SilverWolf_(Maid)",
+  "SilverWolfLora_withhand",
+];
+
+function editSegment(stage) {
+  /** @type {import('../../../web/types/litegraph.js').LGraph}*/
+  const graph = app.graph;
+  const imageNodes = graph.findNodesByType("LoadImage");
+  if (!imageNodes[0].imgs) return;
+
+  const nodes = graph.findNodesByType("SAM MultiLayer");
+
+  /** @type {any[]}*/
+  const widgets = nodes[0].widgets;
+  console.log(nodes[0]);
+  console.log(nodes[0].widgets);
+  widgets.find((x) => x.type == "button").callback();
+  stage.val = 2;
+}
+
+// const workflowList = ["Auto_segment_workflow"];
+/**
+ * Load JSON workflow
+ * @param {string} name - The name of the workflow to load
+ */
+async function loadJSONWorkflow(name) {
+  if (name === 'default' || name.toLowerCase().startsWith("auto_segment")) {
+    enableAutoSegment.val = true
+  } else {
+    enableAutoSegment.val = false
+  }
+  const json = await (await fetch(`./get_workflow?name=${name}`)).json();
+  app.loadGraphData(json);
+  console.log(json);
+}
+
+async function updatePositivePrompt(app, prompt) {
+  const positivePrompt = app.graph
+    .findNodesByType("CLIPTextEncode")
+    .find((x) => x.color == "#232");
+  if (!positivePrompt) {
+    alertDialog.val = {
+      text: "Cannot find the CLIPTextEncode node. Please make sure the workflow is correct.",
+      time: 5000,
+    };
+    return;
+  }
+
+  positivePrompt.widgets[0].inputEl.value = prompt;
+}
+
+async function updateSeedValue(app, seed) {
+  const kSampler = app.graph.findNodesByType("KSampler")[0];
+  if (!kSampler) {
+    alertDialog.val = {
+      text: "Cannot find the KSampler node. Please make sure the workflow is correct.",
+      time: 5000,
+    };
+    return;
+  }
+  kSampler.widgets[0].value = seed;
+  kSampler.widgets[1].value = "fixed";
+}
+
+async function uploadImage() {
+  /** @type {import('../../../web/types/litegraph.js').LGraph}*/
+  const graph = app.graph;
+  const nodes = graph.findNodesByType("LoadImage");
+  previewImgLoading.val = true;
+  console.log(previewImgLoading.val);
+
+  /** @type {any[]}*/
+  const widgets = nodes[0].widgets;
+  console.log(nodes[0]);
+  widgets.find((x) => x.type == "button").callback();
+  while (true) {
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+    if (nodes[0]?.imgs) {
+      if (previewImg.val != "" && previewImg.val == nodes[0].imgs[0].currentSrc)
+        continue;
+      previewImgLoading.val = false;
+      return nodes[0].imgs[0].currentSrc;
+    }
+  }
+}
+
+const jsonWorkflowLoading = van.state(true);
+export const sharedAvatarLink = van.state("");
+
+async function prepareImageFromUrlRedirect(stage) {
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+  const queue_id = new URLSearchParams(window.location.search).get("queue-id");
+  if (queue_id && queue_id != "") {
+    console.log(queue_id);
+    stage.val = 1;
+    const graph = app.graph;
+    const node = graph.findNodesByType("LoadImage");
+    const imageName = queue_id + ".png";
+    console.log(node[0]);
+    node[0].widgets_values[0] = imageName;
+    node[0].widgets[0].value = imageName;
+    node[0].widgets[0]._value = imageName;
+    graph.change();
+    previewImg.val = api.apiURL(
+      `/view?filename=${encodeURIComponent(
+        imageName
+      )}&type=input&subfolder=create_avatar_endpoint${app.getPreviewFormatParam()}`
+    );
+    console.log(previewImg);
+  }
+  const dragndrop = document.getElementById("dnd");
+  dragndrop.addEventListener("dragenter", (evt) => {
+    evt.preventDefault();
+    dragndrop.className =
+      "h-96 w-full border-2 border-purple-500 text-purple-500 border-dashed rounded-lg flex justify-center items-center";
+  });
+  dragndrop.addEventListener("dragleave", (evt) => {
+    evt.preventDefault();
+    dragndrop.className =
+      "h-96 w-full border-2 border-black border-dashed items-center rounded-lg flex justify-center";
+  });
+  dragndrop.addEventListener("dragover", (evt) => {
+    evt.preventDefault();
+  });
+  dragndrop.addEventListener("drop", async (evt) => {
+    evt.preventDefault();
+    dragndrop.className =
+      "h-96 w-full border-2 border-black border-dashed items-center rounded-lg flex justify-center";
+    if (evt.dataTransfer.files.length > 1) return;
+    if (
+      evt.dataTransfer.files[0].type != "image/jpeg" &&
+      evt.dataTransfer.files[0].type != "image/png" &&
+      evt.dataTransfer.files[0].type != "image/webp"
+    )
+      return;
+    stage.val = 1;
+    previewImg.val = URL.createObjectURL(evt.dataTransfer.files[0]);
+    if (Object.entries(evt.dataTransfer.files).length) {
+      await uploadFile(evt.dataTransfer.files[0], true);
+    }
+  });
+}
 
 export function AvatarPreview() {
-  return div(
-    {
-      class: () =>
-        "w-[320px] h-[370px] absolute right-0 top-0 z-[100] pointer-events-auto mt-4 mr-4 " +
-        (!showEditor.val ? "" : "hidden"),
-    },
-    iframe({
+  console.log("getting workflow json now");
+
+  const loading = van.state(false);
+  const shareLoading = van.state("share"); // share, loading, shared
+
+  api.addEventListener("execution_start", (evt) => {
+    loading.val = true;
+  });
+
+  api.addEventListener("executed", (evt) => {
+    const nodeId = evt.detail.node;
+    const targetNode = graph._nodes_by_id[nodeId];
+    if (targetNode.type === "AvatarMainOutput") {
+      loading.val = false;
+    }
+  });
+
+  const email = van.state("");
+  const stage = van.state(0); // 0: upload image, 1: edit segment, 2: generate
+
+  // This will wait 2 seconds until the everything is loaded
+  prepareImageFromUrlRedirect(stage);
+
+  const renderSteps = () => {
+    return div(
+      {
+        class: () =>
+          "flex flex-col bg-white justify-center w-[32rem] max-w-[100%]",
+      },
+      div(
+        {
+          class: () =>
+            " bg-gradient-to-b from-black via-[#5F5F5F] via-60% to-white text-transparent bg-clip-text font-gabarito text-4xl",
+        },
+        "Avatech v1"
+      ),
+      div(
+        {
+          class: () =>
+            " bg-gradient-to-b from-black via-[#5F5F5F] via-50% to-white text-transparent bg-clip-text font-gabarito text-2xl",
+        },
+        "Get your DALLE3 AI Personal Clone"
+      ),
+      div(
+        {
+          class: () =>
+            " w-full flex flex-col justify-center items-center gap-4",
+        },
+        !isGenerateFlow.val
+          ? div(
+            {
+              class: () =>
+                "flex flex-col justify-center items-center gap-4 w-full",
+            },
+            div(
+              { class: () => "w-full flex mt-2" },
+              button(
+                {
+                  class: () => `btn w-full normal-case`,
+                  onclick: async () => {
+                    // previewImg.val = await uploadImage();
+                    // stage.val = 1;
+                    var input = document.createElement("input");
+                    input.type = "file";
+
+                    document.body.appendChild(input);
+
+                    // when the input content changes, do something
+                    input.onchange = async function (e) {
+                      stage.val = 1;
+                      if (Object.entries(e.target.files).length) {
+                        await uploadFile(e.target.files[0], true);
+                      }
+                      previewImg.val = URL.createObjectURL(e.target.files[0]);
+                      // upload files
+                      document.body.removeChild(input);
+                    };
+
+                    // Trigger file browser
+                    input.click();
+                  },
+                },
+                div({ class: "badge badge-neutral" }, "1"),
+                div("Upload your image"),
+                span({
+                  class: "iconify text-lg",
+                  "data-icon": "material-symbols:drive-folder-upload",
+                  "data-inline": "false",
+                }),
+                () =>
+                  previewImgLoading.val
+                    ? span({
+                      class: "loading loading-spinner loading-md",
+                    })
+                    : "",
+              ),
+            ),
+            () => {
+              const dnd = div(
+                {
+                  id: "dnd",
+                  class: () =>
+                    "h-96 w-full border-2 border-black border-dashed items-center rounded-lg flex justify-center text-black",
+                },
+                "or drag and drop the image here",
+              );
+              const image = img({
+                class: () => "z-[10] object-contain w-full h-[394px] border",
+                src: previewImg,
+                onload: () => {
+                  segmented.val = false;
+                }
+              });
+
+              if (isMobileDevice()) {
+                return previewImg.val !== "" ? image : "";
+              } else {
+                return previewImg.val === "" ? dnd : image;
+              }
+            },
+            button(
+              {
+                class: () =>
+                  "btn w-full normal-case " +
+                  (stage.val < 1 ? "btn-disabled" : ""),
+                onclick: () => {
+                  enableAutoSegment.val = true;
+                  editSegment(stage)
+                },
+              },
+              div({ class: "badge badge-neutral" }, "2"),
+              "Edit Segment",
+            ),
+            button(
+              {
+                class: () =>
+                  "btn w-full normal-case " +
+                  (stage.val < 2 ? "btn-disabled" : ""),
+                onclick: async () => {
+                  // const uploaded = await uploadSegments();
+                  // if (!uploaded) return;
+
+                  const graph = app.graph;
+                  const imageNodes = graph.findNodesByType("LoadImage");
+                  if (!imageNodes[0].imgs) return;
+                  document.getElementById("queue-button").click();
+                },
+              },
+              div({ class: "badge badge-neutral" }, "3"),
+              () =>
+                loading.val
+                  ? span({
+                    class: "loading loading-spinner loading-md",
+                  })
+                  : "Make It Alive!",
+            ),
+          )
+          : div(
+            {
+              class:
+                "flex flex-col justify-center items-center gap-4 w-full text-black",
+            },
+            div(
+              {
+                class:
+                  "w-full mt-2 flex flex-col rounded-md left-0 top-0",
+              },
+              textarea({
+                class:
+                  "textarea textarea-bordered border-gray-300 border-b-0 focus:outline-none resize-none rounded-t-md rounded-b-none text-md h-36",
+                placeholder: "Enter your prompt",
+                defaultValue:
+                  "1girl, looking at viewer, open mouth, simple background, white background, smile",
+                id: "positivePromptProxy",
+              }),
+              div(
+                {
+                  class:
+                    "flex flex-row gap-2 border border-gray-300 rounded-b-md text-md items-center",
+                },
+                span({ class: "ml-4" }, "Seed"),
+                div({ class: "divider divider-horizontal m-0" }),
+                input({
+                  type: "text",
+                  class: "input border-none focus:outline-none w-full p-0",
+                  placeholder: "Seed",
+                  defaultValue: "1234",
+                  id: "seedProxy",
+                }),
+                div(
+                  {
+                    onclick: () => {
+                      const random4Digits =
+                        Math.floor(Math.random() * 9000) + 1000;
+                      console.log(
+                        random4Digits,
+                        document.getElementById("seedProxy").value,
+                      );
+                      document.getElementById("seedProxy").value =
+                        random4Digits.toString();
+                    },
+                  },
+                  span({
+                    class: "iconify text-2xl mr-4 hover:cursor-pointer",
+                    "data-icon": "fad:random-1dice",
+                    "data-inline": "false",
+                  }),
+                ),
+              ),
+            ),
+            button(
+              {
+                class: "btn w-full normal-case ",
+                onclick: async () => {
+                  loading.val = true;
+
+                  updatePositivePrompt(
+                    app,
+                    document.getElementById("positivePromptProxy").value,
+                  );
+                  updateSeedValue(
+                    app,
+                    document.getElementById("seedProxy").value,
+                  );
+
+                  const sam = app.graph.findNodesByType("SAM MultiLayer")[0];
+                  if (!sam) {
+                    alertDialog.val = {
+                      text: "Cannot find the SAM node. Please make sure the workflow is correct.",
+                      time: 5000,
+                    };
+                    return;
+                  }
+                  const ckpt = sam.widgets[0].value;
+                  const modelType = ckpt.match(/vit_[lbh]/)?.[0];
+                  await initModel(modelType);
+                  await uploadSegments();
+
+                  document.getElementById("queue-button").click();
+                },
+              },
+              div({ class: "badge badge-neutral" }, "1"),
+              () =>
+                loading.val
+                  ? span({ class: "loading loading-spinner loading-md" })
+                  : "Make It Alive!",
+            ),
+            button(
+              {
+                class: () =>
+                  "btn w-full normal-case ",
+                onclick: () => {
+                  enableAutoSegment.val = false;
+                  editSegment(stage)
+                },
+              },
+              div({ class: "badge badge-neutral" }, "2"),
+              "Edit Segment",
+            ),
+            // button(
+            //   {
+            //     class: "btn w-full normal-case",
+            //     onclick: () => {
+            //       /** @type {import('../../../web/types/litegraph.js').LGraph}*/
+            //       const graph = app.graph;
+            //       const nodes = graph.findNodesByType("SAM MultiLayer");
+
+            //       /** @type {any[]}*/
+            //       const widgets = nodes[0].widgets;
+            //       console.log(nodes[0]);
+            //       console.log(nodes[0].widgets);
+            //       widgets.find((x) => x.type == "button").callback();
+            //     },
+            //   },
+            //   div({ class: "badge badge-neutral" }, "2"),
+            //   "(Optional) Edit Segment",
+            // ),
+          ),
+      ),
+    );
+  };
+
+  const renderIFrame = () => {
+    return iframe({
       id: "avatech-viewer-iframe",
       title: "avatech-viewer-iframe",
       name: "avatech-viewer-iframe",
       allow: "cross-origin-isolated",
       class: () =>
-        "w-full h-full flex pointer-events-auto rounded-2xl border-none " +
-        (!showEditor.val ? "" : "hidden"),
+        "w-full h-full min-w-[350px] min-h-[350px] z-[100] pointer-events-auto flex border-none overflow-hidden bg-transparent" +
+        (showPreview.val ? "" : "hidden"),
       // src: "https://labs.avatech.ai/viewer/default",
       // src: "http://localhost:3000/viewer/default",
       src: previewUrl,
-    }),
+    });
+  };
+
+  const renderShareLink = () => {
+    return div(
+      {
+        class: () =>
+          "w-full flex flex-col gap-2 justify-center items-center mt-8",
+      },
+      div(
+        {
+          class: () =>
+            "w-full flex justify-center font-bold italic text-gray-500",
+        },
+        span("We are launching OpenAI Assistant API integration soon!")
+      ),
+      div(
+        { class: () => "w-[24rem] flex justify-center items-center" },
+        input({
+          type: "text",
+          class: () =>
+            "w-full input input-bordered text-black rounded rounded-l-md rounded-r-none !outline-none",
+          onchange: (e) => {
+            email.val = e.target.value;
+          },
+          placeholder: "Enter your email",
+        }),
+        button(
+          {
+            class: () =>
+              "btn rounded rounded-l-none rounded-r-md no-animation bg-neutral-800 hover:bg-neutral-950 text-white border-none normal-case",
+            onclick: async () => {
+              if (shareLoading.val === "share") {
+                shareLoading.val = "loading";
+                const url = await (await fetch("./get_webhook")).json();
+                await uploadPreview();
+                await fetch(url, {
+                  method: "POST",
+                  headers: {
+                    "Content-Type": "application/json",
+                  },
+                  body: JSON.stringify({
+                    username: "Avabot",
+                    avatar_url:
+                      "https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/avatechai.png",
+                    content: "New register! \n" + email.val,
+                  }),
+                });
+                shareLoading.val = "shared";
+              }
+              if (sharedAvatarLink.val) {
+                await navigator.clipboard.writeText(sharedAvatarLink.val);
+                alertDialog.val = {
+                  text: "Avatar link copied to clipboard!",
+                  type: "success",
+                  time: 5000,
+                };
+              }
+            },
+          },
+          () => {
+            switch (shareLoading.val) {
+              case "share":
+                return "Get Avatar Link";
+              case "loading":
+                return span({
+                  class: "loading loading-spinner loading-md",
+                });
+              case "shared":
+                return span({
+                  class: "iconify text-xl",
+                  "data-icon": "lucide:copy-check",
+                });
+            }
+          }
+        )
+      )
+    );
+  };
+
+  const renderCloseButton = () => {
+    return button(
+      {
+        class: () =>
+          "btn flex flex-row btn-ghost text-black normal-case rounded-md left-0 top-0 z-[200] pointer-events-auto sm:btn-md btn-sm ",
+        onclick: () => {
+          showPreview.val = false;
+        },
+      },
+      span({
+        class: "iconify text-lg",
+        "data-icon": "ic:round-close",
+        "data-inline": "false",
+      })
+    );
+  };
+
+  const renderRestartButton = () => {
+    return button(
+      {
+        class: () =>
+          "btn flex flex-row btn-ghost text-black normal-case rounded-md left-0 top-0 z-[200] pointer-events-auto sm:btn-md btn-sm ",
+        onclick: () => {
+          fetch("https://7a49f4ad27be4dcf.ngrok.app/restart");
+        },
+      },
+      span({
+        class: "iconify text-lg",
+        "data-icon": "mdi:restart",
+        "data-inline": "false",
+      })
+    );
+  };
+
+  const renderChangeWorkflowButton = () => {
+    return div(
+      {
+        class: () =>
+          "dropdown dropdown-hover dropdown-bottom z-[200] pointer-events-auto text-black ",
+      },
+      label(
+        {
+          class: () =>
+            "btn flex flex-row btn-ghost normal-case rounded-md sm:btn-md btn-sm",
+          tabIndex: () => 0,
+        },
+        span({
+          class: "iconify text-lg",
+          "data-icon": "ic:round-swap-vert",
+          "data-inline": "false",
+        }),
+        span({ class: "sm:flex hidden" }, () =>
+          jsonWorkflowLoading.val ? "Loading" : "Change workflow"
+        )
+      ),
+      ul(
+        {
+          class: () =>
+            "dropdown-content -left-[100px] z-[200] menu p-2 shadow rounded-box w-96 bg-white",
+          tabIndex: () => 0,
+        },
+        workflowList.map((val, index) => {
+          return li(
+            {
+              class: () => "p-4 btn btn-ghost items-start",
+              onclick: async (e) => {
+                e.preventDefault();
+                document.activeElement.blur()
+                await loadJSONWorkflow(val);
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                const kSampler = app.graph.findNodesByType("KSampler")[0];
+                if (!kSampler) isGenerateFlow.val = false;
+                else isGenerateFlow.val = true;
+              },
+            },
+            () => val
+          );
+        }),
+        div({ class: () => "divider !my-0" }),
+        li(
+          {
+            class: () => "p-4 btn btn-ghost items-start",
+            onclick: (e) => {
+              let input = document.createElement("input");
+              input.type = "file";
+              document.body.appendChild(input);
+              input.accept = ".json,image/png,.latent,.safetensors";
+              input.onchange = async function (e) {
+                if (Object.entries(e.target.files).length) {
+                  await app.handleFile(e.target.files[0]);
+                }
+                await new Promise((resolve) => setTimeout(resolve, 200));
+                const kSampler = app.graph.findNodesByType("KSampler")[0];
+                if (!kSampler) isGenerateFlow.val = false;
+                else isGenerateFlow.val = true;
+                document.body.removeChild(input);
+              };
+              input.click();
+              // document.getElementById("comfy-load-button").click();
+            },
+          },
+          "Import..."
+        )
+      )
+    );
+    // return button(
+    //   {
+    //     class: () =>
+    //       "btn text-black flex flex-row btn-ghost normal-case rounded-md left-0 top-0 z-[200] pointer-events-auto sm:btn-md btn-sm ",
+    //     onclick: () => {
+    //   let input = document.createElement("input");
+    //   input.type = "file";
+    //   document.body.appendChild(input);
+    //   input.accept = ".json,image/png,.latent,.safetensors";
+    //   input.onchange = async function (e) {
+    //     if (Object.entries(e.target.files).length) {
+    //       await app.handleFile(e.target.files[0]);
+    //     }
+    //     await new Promise((resolve) => setTimeout(resolve, 200));
+    //     const kSampler = app.graph.findNodesByType("KSampler")[0];
+    //     if (!kSampler) isGenerateFlow.val = false;
+    //     else isGenerateFlow.val = true;
+    //     document.body.removeChild(input);
+    //   };
+    //   input.click();
+    //   // document.getElementById("comfy-load-button").click();
+    // },
+    //   },
+    //   span({
+    //     class: "iconify text-lg",
+    //     "data-icon": "ic:round-swap-vert",
+    //     "data-inline": "false",
+    //   }),
+    // span({ class: "sm:flex hidden" }, () =>
+    //   jsonWorkflowLoading.val ? "Loading" : "Change workflow",
+    // ),
+    // );
+  };
+
+  const renderTwitter = () => {
+    return button(
+      {
+        class: () =>
+          "absolute top-4 right-4 btn sm:w-32 w-20 text-black btn-ghost text-xs z-[200] !px-0 normal-case sm:btn-md btn-sm",
+        onclick: () => window.open("https://twitter.com/avatech_gg", "_blank"),
+      },
+      "Twitter"
+    );
+  };
+
+  const isMobileDevice = () => {
+    return window.screen.width < 768;
+  };
+
+  return div(
+    {
+      class: () => {
+        console.log(showPreview);
+
+        return (
+          (showPreview.val ? "" : "hidden ") +
+          "absolute w-[360px] h-[360px] rounded-xl overflow-hidden right-0 top-0 z-[99] pointer-events-auto flex border-none bg-transparent"
+        );
+      },
+    },
+    renderIFrame(),
   );
 }
+
+function showImage(name) {
+  const graph = app.graph;
+  const node = graph.findNodesByType("LoadImage");
+  const img = new Image();
+  img.onload = () => {
+    node[0].imgs = [img];
+    app.graph.setDirtyCanvas(true);
+  };
+  let folder_separator = name.lastIndexOf("/");
+  let subfolder = "";
+  if (folder_separator > -1) {
+    subfolder = name.substring(0, folder_separator);
+    name = name.substring(folder_separator + 1);
+  }
+  img.src = api.apiURL(
+    `/view?filename=${encodeURIComponent(
+      name
+    )}&type=input&subfolder=${subfolder}${app.getPreviewFormatParam()}`
+  );
+  node.setSizeForImage?.();
+}
+
+async function uploadFile(file, updateNode, pasted = false) {
+  try {
+    // Wrap file in formdata so it includes filename
+    const graph = app.graph;
+    const nodes = graph.findNodesByType("LoadImage");
+    const widgets = nodes[0].widgets.find((w) => w.name === "image");
+    const body = new FormData();
+    body.append("image", file);
+    if (pasted) body.append("subfolder", "pasted");
+    const resp = await api.fetchApi("/upload/image", {
+      method: "POST",
+      body,
+    });
+
+    if (resp.status === 200) {
+      const data = await resp.json();
+      // Add the file to the dropdown list and update the widget value
+      let path = data.name;
+      if (data.subfolder) path = data.subfolder + "/" + path;
+
+      if (!widgets.options.values.includes(path)) {
+        widgets.options.values.push(path);
+      }
+
+      if (updateNode) {
+        showImage(path);
+        widgets.value = path;
+      }
+    } else {
+      alert(resp.status + " - " + resp.statusText);
+    }
+  } catch (error) {
+    alert(error);
+  }
+}

js/Container.js

@@ -4,6 +4,7 @@ import { van } from './van.js';
 import { AvatarPreview } from './AvatarPreview.js';
 import { Loading } from './Loading.js';
 import { Alert } from './Alert.js';
+import { AppHeader } from './AppHeader.js';
 const { button, iframe, div, img } = van.tags;
 
 export function Container() {
@@ -16,6 +17,6 @@ export function Container() {
     LayerEditor(),
     AvatarPreview(),
     Loading(),
-    Alert(),
+    Alert()
   );
 }

js/GetShareLink.js

@@ -0,0 +1,29 @@
+import { van } from "./van.js";
+const { button, div, span, input } = van.tags;
+
+export function GetShareLink() {
+  return div(
+    {
+      class: () =>
+        "absolute flex flex-col justify-center items-center top-0 left-0 bg-gray-900 bg-opacity-50 pointer-events-auto w-full h-full gap-2",
+    },
+    span("We're launching OpenAI Assistant API integration soon!"),
+    div(
+      {
+        class: "w-[24rem] flex justify-center items-center",
+      },
+      input({
+        class:
+          "w-full input input-bordered text-black rounded rounded-l-md rounded-r-none",
+        placeholder: "Email",
+      }),
+      button(
+        {
+          class:
+            "btn rounded rounded-l-none rounded-r-md no-animation bg-neutral hover:bg-neutral-focus text-white border-none normal-case",
+        },
+        "Get Avatar Link"
+      )
+    )
+  );
+}

js/LayerEditor.js

@@ -1,5 +1,7 @@
 import { SideBar } from "./SideBar.js";
-import { initModel, runONNX } from "./onnx.js";
+import { api } from "./api.js";
+import { app } from "./app.js";
+import { runONNX } from "./onnx.js";
 import {
   showImageEditor,
   point_label,
@@ -11,11 +13,295 @@ import {
   selectedLayer,
   imagePromptsMulti,
   embeddings,
+  embeddingID,
+  alertDialog,
+  allImagePrompts,
+  boxesMulti,
+  enableAutoSegment,
 } from "./state.js";
 import { van } from "./van.js";
+import vision from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3";
+const { PoseLandmarker, FaceLandmarker, FilesetResolver } = vision;
 const { button, div, img, canvas, span } = van.tags;
 
 let throttle = false;
+const positivePrompt = van.state(true);
+const enableBackgroundRemover = van.state(true);
+const isMobileDevice = () => {
+  return window.screen.width < 768;
+};
+
+// Auto segmentation
+const filesetResolver = await FilesetResolver.forVisionTasks(
+  "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm"
+);
+const faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
+  baseOptions: {
+    modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
+    delegate: "GPU",
+  },
+  // outputFaceBlendshapes: true,
+  runningMode: "IMAGE",
+  numFaces: 1,
+});
+const poseLandmarker = await PoseLandmarker.createFromOptions(filesetResolver, {
+  baseOptions: {
+    modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_full/float16/1/pose_landmarker_full.task`,
+    delegate: "GPU",
+  },
+  runningMode: "IMAGE",
+  numPoses: 1,
+});
+const layerMapping = {
+  L_eye: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 0,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: 0.25,
+    negativeScale: 0.5,
+    indices: FaceLandmarker.FACE_LANDMARKS_LEFT_EYE,
+  },
+  R_eye: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 0,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: 0.25,
+    negativeScale: 0.5,
+    indices: FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE,
+  },
+  L_iris: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 0,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: -0.2,
+    negativeScale: 0.5,
+    indices: FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS,
+  },
+  R_iris: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 0,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: -0.2,
+    negativeScale: 0.5,
+    indices: FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS,
+  },
+  face: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 60,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: 0.5,
+    negativeScale: 0,
+    indices: FaceLandmarker.FACE_LANDMARKS_FACE_OVAL,
+  },
+  mouth: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 0,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: -0.3,
+    negativeScale: 0.3,
+    // https://stackoverflow.com/questions/66649492/how-to-get-specific-landmark-of-face-like-lips-or-eyes-using-tensorflow-js-face
+    indices: [61, 37, 270, 91, 314].map((x) => ({
+      start: x,
+      end: x,
+    })),
+  },
+  mouth_in: {
+    useMiddle: false,
+    positiveOffsetX: 0,
+    positiveOffsetY: 0,
+    negativeOffsetX: 0,
+    negativeOffsetY: 0,
+    positiveScale: -0.5,
+    negativeScale: 0.5,
+    // https://stackoverflow.com/questions/66649492/how-to-get-specific-landmark-of-face-like-lips-or-eyes-using-tensorflow-js-face
+    indices: [310, 88].map((x) => ({
+      start: x,
+      end: x,
+    })),
+  },
+};
+
+export const segmented = van.state(false);
+
+export async function autoSegment() {
+  const image = document.getElementById("image");
+  const landmarks = faceLandmarker.detect(image).faceLandmarks[0];
+
+  Object.entries(layerMapping).forEach(([key, value]) => {
+    imagePromptsMulti.val[key] = [];
+  });
+
+  Object.entries(layerMapping).forEach(([key, value]) => {
+    const positivePoints = [];
+    const middlePoints = [];
+    const negativePoints = [];
+
+    // Positive points
+    for (const { start, end } of value.indices) {
+      const startPoint = landmarks[start];
+      // const endPoint = landmarks[end];
+
+      const startX = startPoint.x * imageSize.val.width;
+      const startY = startPoint.y * imageSize.val.height;
+
+      // const endX = endPoint.x * imageSize.val.width;
+      // const endY = endPoint.y * imageSize.val.height;
+
+      if (middlePoints.length === 0) {
+        middlePoints.push({ x: startX, y: startY, label: 1, isAuto: true });
+        // middlePoints.push({ x: endX, y: endY, label: 1 });
+      } else {
+        middlePoints[0].x += startX;
+        middlePoints[0].y += startY;
+        // middlePoints[1].x += endX;
+        // middlePoints[1].y += endY;
+      }
+      positivePoints.push({ x: startX, y: startY, label: 1, isAuto: true });
+      // positivePoints.push({ x: endX, y: endY, label: 1 });
+
+      // imagePrompts.val = [...imagePrompts.val, { x, y, label: 1 }];
+    }
+
+    // Middle points
+    const len = value.indices.length;
+    middlePoints[0].x /= len;
+    middlePoints[0].y /= len;
+    // middlePoints[1].x /= len;
+    // middlePoints[1].y /= len;
+
+    if (value.useMiddle) {
+      imagePromptsMulti.val[key] = [
+        ...imagePromptsMulti.val[key],
+        ...middlePoints,
+      ];
+    } else {
+      // Negative points
+      for (const [i, { start, end }] of value.indices.entries()) {
+        const startPoint = landmarks[start];
+        // const endPoint = landmarks[end];
+
+        const startX = startPoint.x * imageSize.val.width;
+        const startY = startPoint.y * imageSize.val.height;
+
+        // const endX = endPoint.x * imageSize.val.width;
+        // const endY = endPoint.y * imageSize.val.height;
+
+        const middlePoint = middlePoints[0];
+        const directionVector = {
+          x: middlePoint.x - startX,
+          y: middlePoint.y - startY,
+        };
+        const directionVectorLength = Math.sqrt(
+          directionVector.x * directionVector.x +
+            directionVector.y * directionVector.y
+        );
+
+        if (value.negativeScale !== 0) {
+          const negativePointDistance =
+            value.negativeScale * directionVectorLength;
+          const negativePoint = {
+            x:
+              startX -
+              (negativePointDistance * directionVector.x) /
+                directionVectorLength -
+              value.negativeOffsetX,
+            y:
+              startY -
+              (negativePointDistance * directionVector.y) /
+                directionVectorLength -
+              value.negativeOffsetY,
+            label: 0,
+            isAuto: true,
+          };
+          negativePoints.push(negativePoint);
+        }
+
+        const positivePointDistance =
+          value.positiveScale * directionVectorLength;
+        positivePoints[i] = {
+          x:
+            positivePoints[i].x -
+            (positivePointDistance * directionVector.x) /
+              directionVectorLength -
+            value.positiveOffsetX,
+          y:
+            positivePoints[i].y -
+            (positivePointDistance * directionVector.y) /
+              directionVectorLength -
+            value.positiveOffsetY,
+          label: 1,
+          isAuto: true,
+        };
+      }
+      imagePromptsMulti.val[key] = [
+        ...imagePromptsMulti.val[key],
+        ...positivePoints,
+        ...negativePoints,
+      ];
+    }
+
+    // Find bounding box of positive/negative points
+    const points = negativePoints.length > 0 ? negativePoints : positivePoints;
+    const box = {
+      x1: Math.min(...points.map((x) => x.x)),
+      y1: Math.min(...points.map((x) => x.y)),
+      x2: Math.max(...points.map((x) => x.x)),
+      y2: Math.max(...points.map((x) => x.y)),
+    };
+    boxesMulti.val[key] = box;
+  });
+
+  const poseLandmarks = poseLandmarker.detect(image).landmarks[0];
+  const positiveBreathX =
+    ((poseLandmarks[11].x + poseLandmarks[12].x) / 2) * imageSize.val.width;
+  const positiveBreathY =
+    ((poseLandmarks[11].y + poseLandmarks[12].y) / 2) * imageSize.val.height;
+  const negativeBreathX1 = poseLandmarks[0].x * imageSize.val.width;
+  const negativeBreathY1 = poseLandmarks[0].y * imageSize.val.height;
+  const negativeBreathX2 = poseLandmarks[9].x * imageSize.val.width;
+  const negativeBreathY2 = poseLandmarks[9].y * imageSize.val.height;
+  const negativeBreathX3 = poseLandmarks[10].x * imageSize.val.width;
+  const negativeBreathY3 = poseLandmarks[10].y * imageSize.val.height;
+
+  imagePromptsMulti.val["breath"] = [
+    { x: positiveBreathX, y: positiveBreathY, label: 1, isAuto: true },
+    { x: negativeBreathX1, y: negativeBreathY1, label: 0, isAuto: true },
+    { x: negativeBreathX2, y: negativeBreathY2, label: 0, isAuto: true },
+    { x: negativeBreathX3, y: negativeBreathY3, label: 0, isAuto: true },
+  ];
+  imagePrompts.val = imagePromptsMulti.val[selectedLayer.val];
+  segmented.val = true;
+  console.log("Done");
+}
+
+export function setRemoveBackgroundNode() {
+  const rmBgNodes = app.graph.findNodesByType(
+    "Image Rembg (Remove Background)"
+  );
+  if (!rmBgNodes?.length) {
+    alertDialog.val = {
+      text: "Remove background node not found. Please ensure the workflow is correct.",
+      time: 5000,
+    };
+    return;
+  }
+  rmBgNodes.forEach((node) => {
+    // node is bypassed if mode is 4
+    node.mode = enableBackgroundRemover.val ? 0 : 4;
+  });
+}
 
 export function updateImagePrompts() {
   if (selectedLayer.val !== "" && selectedLayer.val !== undefined) {
@@ -26,6 +312,18 @@ export function updateImagePrompts() {
 
     targetNode.val.widgets.find((x) => x.name === "image_prompts_json").value =
       JSON.stringify(imagePromptsMulti.val);
+
+    // const canvas = document.getElementById("mask-canvas");
+    // const base64Image = canvas.toDataURL();
+    // api.fetchApi("/segments", {
+    //   method: "POST",
+    //   body: JSON.stringify({
+    //     name: embeddingID.val,
+    //     segments: {
+    //       [selectedLayer.val]: base64Image,
+    //     },
+    //   }),
+    // });
   } else {
     targetNode.val.widgets.find((x) => x.name === "image_prompts_json").value =
       JSON.stringify(imagePrompts.val);
@@ -33,7 +331,44 @@ export function updateImagePrompts() {
   targetNode.val.graph.change();
 }
 
-function handleClick(e) {
+export async function uploadSegments() {
+  const emptyLayers = [];
+  Object.entries(imagePromptsMulti.val).forEach(([key, value]) => {
+    if (value.length === 0) {
+      emptyLayers.push(key);
+    }
+  });
+  if (emptyLayers.length > 0) {
+    alertDialog.val = {
+      text: "The following layers have no segments: " + emptyLayers.join(", "),
+      time: 5000,
+    };
+    return false;
+  }
+
+  const segments = {};
+  for (const [layer, prompts] of Object.entries(imagePromptsMulti.val)) {
+    await drawSegment(getClicks(prompts), layer, false);
+    const canvas = document.getElementById("mask-canvas");
+    const base64Image = canvas.toDataURL();
+    segments[layer] = base64Image;
+    // download image
+    // const a = document.createElement("a");
+    // a.href = base64Image;
+    // a.download = layer + ".png";
+    // a.click();
+  }
+  await api.fetchApi("/segments", {
+    method: "POST",
+    body: JSON.stringify({
+      name: embeddingID.val,
+      segments,
+    }),
+  });
+  return true;
+}
+
+async function handleClick(e) {
   const rect = e.target.getBoundingClientRect();
   const x = e.clientX - rect.left;
   const y = e.clientY - rect.top;
@@ -45,22 +380,28 @@ function handleClick(e) {
       imageSize.val.imgScale
   );
 
+  let label;
+  if (isMobileDevice()) {
+    label = positivePrompt.val ? 1 : 0;
+  } else {
+    label = e.isRight ? 0 : 1;
+  }
+
   imagePrompts.val = [
     ...imagePrompts.val,
-    { x: relativeX, y: relativeY, label: e.isRight ? 0 : 1 },
+    { x: relativeX, y: relativeY, label },
   ];
-
+  await drawSegment(getClicks());
   updateImagePrompts();
-  drawSegment(getClicks());
 }
 
-function handlePointClick(e, point) {
+async function handlePointClick(e, point) {
   e.preventDefault();
   imagePrompts.val = imagePrompts.val.filter(
     (x) => !(x.x === point.x && x.y === point.y)
   );
+  await drawSegment(getClicks());
   updateImagePrompts();
-  drawSegment(getClicks());
 }
 
 function handleImageSize(image) {
@@ -74,15 +415,16 @@ function handleImageSize(image) {
   return { height: h, width: w, samScale, imgScale };
 }
 
-export function getClicks() {
-  return imagePrompts.val.map((point) => ({
+export function getClicks(prompts) {
+  return (prompts || imagePrompts.val).map((point) => ({
     x: point.x,
     y: point.y,
     clickType: point.label,
+    isAuto: point.isAuto,
   }));
 }
 
-export function drawSegment(clicks) {
+export async function drawSegment(clicks, layer, drawBox = true) {
   const canvas = document.getElementById("mask-canvas");
   const ctx = canvas.getContext("2d");
   if (clicks.length === 0) {
@@ -90,19 +432,34 @@ export function drawSegment(clicks) {
     return;
   }
   if (embeddings.val) {
-    runONNX(clicks, embeddings.val).then((mask) => {
-      if (mask) {
-        ctx.clearRect(0, 0, canvas.width, canvas.height);
-        ctx.drawImage(mask, 0, 0);
+    const box = enableAutoSegment.val
+      ? boxesMulti.val[layer || selectedLayer.val]
+      : null;
+    const filteredClicks = enableAutoSegment.val
+      ? clicks
+      : clicks.filter((click) => !click.isAuto);
+    if (filteredClicks.length === 0) {
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      return;
+    }
+    const mask = await runONNX(filteredClicks, embeddings.val, box);
+    if (mask) {
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
+      ctx.drawImage(mask, 0, 0);
+      if (box && drawBox) {
+        ctx.strokeStyle = "green";
+        ctx.lineWidth = 5;
+        ctx.strokeRect(box.x1, box.y1, box.x2 - box.x1, box.y2 - box.y1);
       }
-    });
+    }
   }
 }
 
-initModel();
-
 export function LayerEditor() {
   let realTimeSegment = true;
+
+  const showSidebar = van.state(true);
+
   document.addEventListener("keydown", (e) => {
     if (showImageEditor.val && e.code === "Tab") {
       e.preventDefault();
@@ -116,29 +473,97 @@ export function LayerEditor() {
   return div(
     {
       class: () =>
-        "absolute flex bg-gray-900 bg-opacity-50 top-0 w-full h-full pointer-events-auto " +
+        "absolute flex bg-gray-900 bg-opacity-50 top-0 w-full h-full pointer-events-auto z-[1000] " +
         (showImageEditor.val ? "" : "hidden"),
     },
-    button(
+    div(
       {
-        class: () =>
-          "btn btn-circle flex flex-row btn-ghost normal-case absolute p-0 rounded-md left-2 top-0 z-[200] w-fit",
-        onclick: () => {
-          console.log("close");
-          showImageEditor.val = false;
-        },
+        class:
+          "absolute top-4 left-4 right-0 flex w-full gap-2 justify-start z-[200]",
       },
-      span({
-        class: "iconify text-lg",
-        "data-icon": "ic:baseline-arrow-back",
-        "data-inline": "false",
-      }),
-      div("Back")
+      button(
+        {
+          class: () => "btn btn-neutral flex flex-row normal-case rounded-md",
+          onclick: async () => {
+            console.log("close");
+            showImageEditor.val = false;
+            await uploadSegments();
+
+            const isEqual = allImagePrompts.val.map(
+              (x) =>
+                JSON.stringify(imagePromptsMulti.val) ===
+                JSON.stringify(x.prompt)
+            );
+            if (!isEqual.includes(true))
+              allImagePrompts.val = [
+                ...allImagePrompts.val,
+                {
+                  version: "v" + allImagePrompts.val.length,
+                  prompt: imagePromptsMulti.val,
+                },
+              ];
+
+            // api.fetchApi("/segments_order", {
+            //   method: "POST",
+            //   body: JSON.stringify({
+            //     name: embeddingID.val,
+            //     order: Object.keys(imagePromptsMulti.val),
+            //   }),
+            // });
+          },
+        },
+        span({
+          class: "iconify text-lg",
+          "data-icon": "ic:baseline-arrow-back",
+          "data-inline": "false",
+        }),
+        div("Back")
+      ),
+      button(
+        {
+          class: () => "btn btn-neutral flex flex-row normal-case rounded-md",
+          onclick: () => (showSidebar.val = !showSidebar.val),
+        },
+        div(() => (showSidebar.val ? "Hide UI" : "Show UI"))
+      ),
+      button(
+        {
+          class: () => "btn btn-neutral flex flex-row normal-case rounded-md",
+          onclick: () => {
+            enableAutoSegment.val = !enableAutoSegment.val;
+            drawSegment(getClicks());
+          },
+        },
+        () => (enableAutoSegment.val ? "Auto Segment On" : "Auto Segment Off")
+      ),
+      button(
+        {
+          class: () => "btn btn-neutral flex flex-row normal-case rounded-md",
+          onclick: () => {
+            enableBackgroundRemover.val = !enableBackgroundRemover.val;
+            setRemoveBackgroundNode();
+          },
+        },
+        () =>
+          enableBackgroundRemover.val
+            ? "Background Remover On"
+            : "Background Remover Off"
+      ),
+      button(
+        {
+          class: () =>
+            `btn btn-neutral flex flex-row normal-case rounded-md ${
+              isMobileDevice() ? "" : "hidden"
+            }`,
+          onclick: () => (positivePrompt.val = !positivePrompt.val),
+        },
+        div(() => (positivePrompt.val ? "Positive" : "Negative"))
+      )
     ),
     div(
       {
         class:
-          "hidden w-full flex justify-center absolute top-0 left-0 right-0 items-center",
+          "hidden w-full justify-center absolute top-0 left-0 right-0 items-center",
       },
       button(
         {
@@ -165,10 +590,11 @@ export function LayerEditor() {
         id: "image-container",
       },
       img({
+        id: "image",
         class:
           "fixed top-1/2 left-1/2 transform -translate-x-1/2 -translate-y-1/2",
         src: imageUrl,
-        onload: (e) => {
+        onload: async (e) => {
           imageSize.val = handleImageSize(e.target);
 
           document.getElementById("image-container").style.scale =
@@ -183,13 +609,13 @@ export function LayerEditor() {
           canvas.width = e.target.naturalWidth;
           canvas.height = e.target.naturalHeight;
         },
-        oncontextmenu: (e) => {
+        oncontextmenu: async (e) => {
           e.preventDefault();
           e.isRight = true;
-          handleClick(e);
+          await handleClick(e);
         },
-        onclick: (e) => {
-          handleClick(e);
+        onclick: async (e) => {
+          await handleClick(e);
         },
         onmouseleave: (e) => {
           drawSegment(getClicks());
@@ -223,19 +649,25 @@ export function LayerEditor() {
           }
         },
       }),
-      canvas({
-        class:
-          "pointer-events-none fixed top-1/2 left-1/2 transform -translate-x-1/2 -translate-y-1/2 opacity-80",
-        id: "mask-canvas",
-      }),
-      () => {
-        return div(
+      () =>
+        canvas({
+          class:
+            "pointer-events-none fixed top-1/2 left-1/2 transform -translate-x-1/2 -translate-y-1/2 opacity-80",
+          style: () =>
+            `width: ${imageContainerSize.val.width}px; height: ${imageContainerSize.val.height}px;`,
+          id: "mask-canvas",
+        }),
+      () =>
+        div(
           {
             class: "absolute w-full h-full pointer-events-none",
             style: () =>
               `width: ${imageContainerSize.val.width}px; height: ${imageContainerSize.val.height}px;`,
           },
-          ...imagePrompts.val?.map((point) => {
+          ...(enableAutoSegment.val
+            ? imagePrompts.val
+            : imagePrompts.val?.filter((click) => !click.isAuto)
+          ).map((point) => {
             return button({
               style: () =>
                 `left: ${
@@ -249,17 +681,16 @@ export function LayerEditor() {
                   point.label === 1 ? "bg-green-500" : "bg-red-500"
                 }`,
 
-              oncontextmenu: (e) => {
-                handlePointClick(e, point);
+              oncontextmenu: async (e) => {
+                await handlePointClick(e, point);
               },
-              onclick: (e) => {
-                handlePointClick(e, point);
+              onclick: async (e) => {
+                await handlePointClick(e, point);
               },
             });
           })
-        );
-      }
+        )
     ),
-    SideBar()
+    () => (showSidebar.val ? SideBar() : div())
   );
 }

js/Loading.js

@@ -6,7 +6,7 @@ export function Loading() {
   return div(
     {
       class: () =>
-        "absolute flex flex-col justify-center items-center top-0 left-0 bg-gray-900 bg-opacity-50 pointer-events-auto w-full h-full " +
+        "absolute flex flex-col justify-center items-center top-0 left-0 bg-gray-900 bg-opacity-50 pointer-events-auto w-full h-full z-[1001] " +
         (showLoading.val ? "" : "hidden"),
     },
     span({

js/ShapeFlowEditor.js

@@ -42,7 +42,7 @@ export function ShapeFlowEditor() {
         {
           class: "modal-box",
         },
-        div({ class: "text-black" }, "This is a dialog"),
+        div({ class: "text-black" }, div({ class: "text-xl font-bold" }, "Shape Flow Editor"), div({ class: "" }, "The shape flow will be save in CreateShapeFlow node (comfyui node)! Or discard the changes!")),
         div(
           { class: "modal-action" },
           form(

js/SideBar.js

@@ -5,6 +5,7 @@ import {
   imagePromptsMulti,
   targetNode,
   showImageEditor,
+  allImagePrompts,
 } from "./state.js";
 import { van } from "./van.js";
 const {
@@ -27,7 +28,8 @@ van.derive(() => {
   if (
     showImageEditor.val &&
     targetNode.val != undefined &&
-    targetNode.val.outputs && targetNode.val.type === 'SAM'
+    targetNode.val.outputs &&
+    targetNode.val.type === "SAM MultiLayer"
   ) {
     const outputNames = targetNode.val.outputs.map((x) => x.name).slice(1);
     const record = Object.keys(imagePromptsMulti.val);
@@ -48,7 +50,7 @@ van.derive(() => {
       missingDiff.forEach((x) => {
         targetNode.val.addOutput(
           x,
-          targetNode.val.type === "SAM" ? "IMAGE" : "SAM_PROMPT"
+          targetNode.val.type === "SAM MultiLayer" ? "IMAGE" : "SAM_PROMPT"
         );
       });
       targetNode.val.graph.change();
@@ -60,185 +62,232 @@ export function SideBar() {
   const layer_to_delete = van.state("");
 
   return div(
-    {
-      class:
-        "ml-2 z-100 w-fit flex-col flex justify-center absolute top-0 left-0 bottom-0 items-start gap-2",
-    },
-
-    () => {
-      const layers = Object.entries(imagePromptsMulti.val);
-      return ul(
-        {
-          class: "menu bg-base-200 w-56 rounded-box text-base-content ",
-        },
-        layers.length === 0 ? li(a("Empty layer")) : null,
-        ...layers.map(([key, value]) => {
-          return li(
-            a(
-              {
-                class: () =>
-                  `normal-case text-start items-start flex items-center justify-between  ${
-                    selectedLayer.val === key ? "active" : ""
-                  }`,
-                onclick: () => {
-                  selectedLayer.val = key;
-                  imagePrompts.val = imagePromptsMulti.val[key];
-                  drawSegment(getClicks());
-                },
-              },
-              key,
-              div(
-                {},
-                button(
-                  {
-                    class:
-                      "btn btn-circle btn-xs btn-ghost group hover:text-red-500",
-                    onclick: (e) => {
-                      console.log("clear");
-                      e.preventDefault();
-                      e.stopPropagation();
-                      imagePrompts.val = [];
-                      imagePromptsMulti.val[key] = [];
-                      drawSegment([]);
-                      updateImagePrompts();
-                    },
-                  },
-                  span({
-                    class: "iconify",
-                    "data-icon": "ant-design:clear-outlined",
-                    "data-inline": "false",
-                  })
-                ),
-                button(
-                  {
-                    class:
-                      "btn btn-circle btn-xs btn-ghost group hover:text-red-500",
-                    onclick: (e) => {
-                      console.log("delete");
-                      e.preventDefault();
-                      e.stopPropagation();
-                      layer_to_delete.val = key;
-                      setTimeout(() => {
-                        delete_layer_dialog.showModal();
-                      }, 0);
-                    },
-                  },
-                  span({
-                    class: "iconify",
-                    "data-icon": "ic:baseline-delete",
-                    "data-inline": "false",
-                  })
-                )
-              )
-            )
-          );
-        }),
-        div({ class: "divider !py-0 my-0" }),
-        li(
-          a(
-            {
-              class: "flex items-center justify-between",
-              onclick: () => {
-                my_modal_3.showModal();
-              },
-            },
-            "New Layer",
-            span({
-              class: "iconify",
-              "data-icon": "ic:outline-plus",
-              "data-inline": "false",
-            })
-          )
-        )
-      );
-    },
-    () =>
-      ConfirmDialog(
-        {
-          id: "delete_layer_dialog",
-          title: "Delete Layer: " + layer_to_delete.val,
-          onsubmit: () => {
-            imagePromptsMulti.val = Object.fromEntries(
-              Object.entries(imagePromptsMulti.val).filter(
-                ([key, value]) => key !== layer_to_delete.val
-              )
-            );
-            console.log(imagePromptsMulti.val);
-            if (selectedLayer.val === layer_to_delete.val) {
-              // Select another layer if there is one
-              if (Object.keys(imagePromptsMulti.val).length > 0) {
-                selectedLayer.val = Object.keys(imagePromptsMulti.val)[0];
-                imagePrompts.val = imagePromptsMulti.val[selectedLayer.val];
-              } else {
-                selectedLayer.val = "";
-                imagePrompts.val = [];
-              }
-            }
-            targetNode.val.graph.change();
-            updateImagePrompts();
-            delete_layer_dialog.close();
+    div(
+      {
+        class:
+          "ml-2 z-100 w-fit flex-col flex justify-center absolute top-0 left-0 bottom-0 items-start gap-2",
+      },
+      () => {
+        const layers = Object.entries(imagePromptsMulti.val);
+        return ul(
+          {
+            class: "menu bg-base-200 w-56 rounded-box text-base-content ",
           },
-        },
-        p("Are you sure you want to delete this layer?")
-      ),
-    () =>
-      dialog(
-        { id: "my_modal_3", class: "modal" },
-        div(
-          { class: "modal-box text-base-content" },
-          form(
+          button(
             {
-              class: "gap-2 flex flex-col",
-              method: "dialog",
-              onsubmit: (e) => {
-                console.log("add new layer");
-                e.preventDefault();
-                const inputText = e.target.elements[1].value;
-                imagePromptsMulti.val = {
-                  ...imagePromptsMulti.val,
-                  [inputText]: [],
-                };
-                console.log(inputText, imagePromptsMulti.val);
-                my_modal_3.close();
-                e.target.elements[1].value = "";
-
-                selectedLayer.val = inputText;
-                imagePrompts.val = imagePromptsMulti.val[inputText];
-                drawSegment(getClicks());
+              onclick: () => {
+                layers.map(([key, value]) => {
+                  imagePrompts.val = [];
+                  imagePromptsMulti.val[key] = [];
+                });
+                drawSegment([]);
                 updateImagePrompts();
               },
+              class: "btn btn-ghost normal-case flex",
             },
-            button(
+            "Clear ALL"
+          ),
+          layers.length === 0 ? li(a("Empty layer")) : null,
+          ...layers.map(([key, value]) => {
+            return li(
+              a(
+                {
+                  class: () =>
+                    `normal-case text-start items-start flex items-center justify-between  ${
+                      selectedLayer.val === key ? "active" : ""
+                    }`,
+                  onclick: () => {
+                    selectedLayer.val = key;
+                    imagePrompts.val = imagePromptsMulti.val[key];
+                    drawSegment(getClicks());
+                  },
+                },
+                key,
+                div(
+                  {},
+                  button(
+                    {
+                      class:
+                        "btn btn-circle btn-xs btn-ghost group hover:text-red-500",
+                      onclick: (e) => {
+                        console.log("clear");
+                        e.preventDefault();
+                        e.stopPropagation();
+                        imagePrompts.val = [];
+                        imagePromptsMulti.val[key] = [];
+                        drawSegment([]);
+                        updateImagePrompts();
+                      },
+                    },
+                    span({
+                      class: "iconify",
+                      "data-icon": "ant-design:clear-outlined",
+                      "data-inline": "false",
+                    })
+                  ),
+                  button(
+                    {
+                      class:
+                        "btn btn-circle btn-xs btn-ghost group hover:text-red-500",
+                      onclick: (e) => {
+                        console.log("delete");
+                        e.preventDefault();
+                        e.stopPropagation();
+                        layer_to_delete.val = key;
+                        setTimeout(() => {
+                          delete_layer_dialog.showModal();
+                        }, 0);
+                      },
+                    },
+                    span({
+                      class: "iconify",
+                      "data-icon": "ic:baseline-delete",
+                      "data-inline": "false",
+                    })
+                  )
+                )
+              )
+            );
+          }),
+          div({ class: "divider !py-0 my-0" }),
+          li(
+            a(
               {
-                type: "button",
-                class: "btn btn-sm btn-circle btn-ghost absolute right-2 top-2",
-                onclick: (e) => {
-                  e.stopPropagation();
-                  my_modal_3.close();
+                class: "flex items-center justify-between",
+                onclick: () => {
+                  my_modal_3.showModal();
                 },
               },
-              "✕"
-            ),
-            h3(
-              { class: "font-bold text-lg text-base-content" },
-              "Add new layer!"
-            ),
-            input({
-              type: "text",
-              placeholder: "Type here",
-              class: "input input-bordered w-full",
-              autofocus: true,
-            }),
-            button(
+              "New Layer",
+              span({
+                class: "iconify",
+                "data-icon": "ic:outline-plus",
+                "data-inline": "false",
+              })
+            )
+          )
+        );
+      },
+      () =>
+        ConfirmDialog(
+          {
+            id: "delete_layer_dialog",
+            title: "Delete Layer: " + layer_to_delete.val,
+            onsubmit: () => {
+              imagePromptsMulti.val = Object.fromEntries(
+                Object.entries(imagePromptsMulti.val).filter(
+                  ([key, value]) => key !== layer_to_delete.val
+                )
+              );
+              console.log(imagePromptsMulti.val);
+              if (selectedLayer.val === layer_to_delete.val) {
+                // Select another layer if there is one
+                if (Object.keys(imagePromptsMulti.val).length > 0) {
+                  selectedLayer.val = Object.keys(imagePromptsMulti.val)[0];
+                  imagePrompts.val = imagePromptsMulti.val[selectedLayer.val];
+                } else {
+                  selectedLayer.val = "";
+                  imagePrompts.val = [];
+                }
+              }
+              targetNode.val.graph.change();
+              updateImagePrompts();
+              delete_layer_dialog.close();
+            },
+          },
+          p("Are you sure you want to delete this layer?")
+        ),
+      () =>
+        dialog(
+          { id: "my_modal_3", class: "modal" },
+          div(
+            { class: "modal-box text-base-content" },
+            form(
               {
-                type: "submit",
-                class: "btn btn-sm  btn-ghost place-self-end",
+                class: "gap-2 flex flex-col",
+                method: "dialog",
+                onsubmit: (e) => {
+                  console.log("add new layer");
+                  e.preventDefault();
+                  const inputText = e.target.elements[1].value;
+                  imagePromptsMulti.val = {
+                    ...imagePromptsMulti.val,
+                    [inputText]: [],
+                  };
+                  console.log(inputText, imagePromptsMulti.val);
+                  my_modal_3.close();
+                  e.target.elements[1].value = "";
+
+                  selectedLayer.val = inputText;
+                  imagePrompts.val = imagePromptsMulti.val[inputText];
+                  drawSegment(getClicks());
+                  updateImagePrompts();
+                },
               },
-              "Confirm"
+              button(
+                {
+                  type: "button",
+                  class:
+                    "btn btn-sm btn-circle btn-ghost absolute right-2 top-2",
+                  onclick: (e) => {
+                    e.stopPropagation();
+                    my_modal_3.close();
+                  },
+                },
+                "✕"
+              ),
+              h3(
+                { class: "font-bold text-lg text-base-content" },
+                "Add new layer!"
+              ),
+              input({
+                type: "text",
+                placeholder: "Type here",
+                class: "input input-bordered w-full",
+                autofocus: true,
+              }),
+              button(
+                {
+                  type: "submit",
+                  class: "btn btn-sm  btn-ghost place-self-end",
+                },
+                "Confirm"
+              )
             )
           )
         )
-      )
+    ),
+    div(
+      {
+        class:
+          "ml-2 z-100 w-fit flex-col flex justify-center absolute top-0 right-0 bottom-0 items-start gap-2 bg-transparent",
+      },
+      () => {
+        return ul(
+          {
+            class: "menu bg-base-200 w-56 rounded-box text-base-content ",
+          },
+          span("Segment History"),
+          ...allImagePrompts.val.map((e) =>
+            li(
+              a(
+                {
+                  class:
+                    "normal-case text-start flex items-center justify-between",
+                  onclick: async () => {
+                    imagePromptsMulti.val = e.prompt;
+                    imagePrompts.val = imagePromptsMulti.val[selectedLayer.val];
+                    drawSegment(getClicks());
+                    updateImagePrompts();
+                  },
+                },
+                e.version
+              )
+            )
+          )
+        );
+      }
+    )
   );
 }
 

js/dialog.js

@@ -0,0 +1,33 @@
+import { ComfyDialog, $el } from '../../scripts/ui.js';
+
+export class InfoDialog extends ComfyDialog {
+	constructor() {
+		super();
+		this.element.classList.add("comfy-normal-modal");
+	}
+	createButtons() {
+		return [
+			$el("button", {
+				type: "button",
+				textContent: "Close",
+				onclick: () => this.close(),
+			}),
+		];
+	}
+
+	close() {
+		this.element.style.display = "none";
+	}
+
+	show(html) {
+		if (typeof html === "string") {
+			this.textElement.innerHTML = html;
+		} else {
+			this.textElement.replaceChildren(html);
+		}
+		this.element.style.display = "flex";
+		this.element.style.zIndex = 1001;
+	}
+}
+
+export const infoDialog = new InfoDialog()

js/index.js

@@ -12,20 +12,35 @@ import {
   showLoading,
   loadingCaption,
   alertDialog,
+  showPreview,
+  shareLoading,
+  previewModelId,
+  embeddingID,
+  enableAutoSegment,
 } from "./state.js";
 import { van } from "./van.js";
 import { app } from "./app.js";
 import { api } from "./api.js";
 import { Container } from "./Container.js";
-import { loadNpyTensor } from "./onnx.js";
+import { initModel, loadNpyTensor } from "./onnx.js";
 import "https://code.iconify.design/3/3.1.0/iconify.min.js";
-import { drawSegment, getClicks } from "./LayerEditor.js";
+import {
+  autoSegment,
+  drawSegment,
+  getClicks,
+  segmented,
+} from "./LayerEditor.js";
+import { infoDialog } from "./dialog.js";
+import { sharedAvatarLink } from "./AvatarPreview.js";
+import { updateImagePrompts } from "./LayerEditor.js";
 
-const stylesheet = document.createElement('link')
-stylesheet.setAttribute('type', "text/css")
-stylesheet.setAttribute('rel', "stylesheet")
-stylesheet.setAttribute('href', './avatar-graph-comfyui/tw-styles.css')
-document.head.appendChild(stylesheet)
+export const generatedImages = {};
+
+const stylesheet = document.createElement("link");
+stylesheet.setAttribute("type", "text/css");
+stylesheet.setAttribute("rel", "stylesheet");
+stylesheet.setAttribute("href", "./avatar-graph-comfyui/tw-styles.css");
+document.head.appendChild(stylesheet);
 
 /** @type {import( '../../../web/types/litegraph.js').LGraphGroup} */
 const recomputeInsideNodesOps = LGraphGroup.prototype.recomputeInsideNodes;
@@ -231,15 +246,35 @@ function getInputWidgetValue(node, inputIndex, widgetName) {
   /** @type {LGraphNode} */
   let nodea = graph._nodes_by_id[targetLink.origin_id];
 
-  while (nodea.type == "Reroute") {
+  while (nodea.type === "Reroute") {
     nodea = nodea.getInputNode(0);
   }
 
   console.log(targetLink, nodea);
-  console.log(nodea.getInputNode(0, true));
 
+  if (nodea.type === "LoadImage") {
+    /** @type {string} */
+    const isGeneratedImage = false;
+    return [
+      isGeneratedImage,
+      nodea.widgets.find((x) => x.name === widgetName).value,
+    ];
+  }
+
+  const saveImageNodeLink = nodea.outputs
+    .find((x) => x.type === "IMAGE")
+    .links.find((link) => {
+      const targetLink = graph.links[link];
+      const targetNode = graph._nodes_by_id[targetLink.target_id];
+      if (targetNode.type === "SaveImage") {
+        return true;
+      }
+    });
+  const saveImageNode =
+    graph._nodes_by_id[graph.links[saveImageNodeLink].target_id];
   /** @type {string} */
-  return nodea.widgets.find((x) => x.name === widgetName).value;
+  const isGeneratedImage = true;
+  return [isGeneratedImage, generatedImages[saveImageNode.id]];
 }
 
 /**
@@ -247,45 +282,88 @@ function getInputWidgetValue(node, inputIndex, widgetName) {
  * @param {LGraphNode} node
  */
 function showMyImageEditor(node) {
-  let connectedImageFileName = getInputWidgetValue(node, 0, "image");
-  const split = connectedImageFileName.split("/");
-  if (split.length > 1) connectedImageFileName = split[1];
-
-  const embeddingFilename = node.widgets.find(
-    (x) => x.name === "embedding_id"
-  ).value;
-
-  const v = JSON.parse(
-    node.widgets.find((x) => x.name === "image_prompts_json").value
+  let [isGeneratedImage, connectedImageFileName] = getInputWidgetValue(
+    node,
+    0,
+    "image"
   );
-
-  if (!Array.isArray(v)) {
-    // this is a multi prompt
-    imagePromptsMulti.val = v;
-    selectedLayer.val = Object.keys(imagePromptsMulti.val)[0];
-    imagePrompts.val = imagePromptsMulti.val[selectedLayer.val];
-  } else {
-    // this is a single prompt
-    selectedLayer.val = "";
-    imagePromptsMulti.val = {};
-    imagePrompts.val = v;
+  if (!connectedImageFileName) {
+    alertDialog.val = {
+      text: "Please connect or generate an image first",
+      time: 3000,
+    };
+    return;
   }
-  showImageEditor.val = true;
-  imageUrl.val = api.apiURL(
-    `/view?filename=${encodeURIComponent(
-      connectedImageFileName
-    )}&type=input&subfolder=${split.length > 1 ? split[0] : ""}`
-  );
-  const embeedingUrl = api.apiURL(
-    `/view?filename=${encodeURIComponent(
-      `${embeddingFilename}.npy`
-    )}&type=output&subfolder=`
-  );
-  loadNpyTensor(embeedingUrl).then((tensor) => {
-    embeddings.val = tensor;
-    drawSegment(getClicks());
+
+  loadingCaption.val = "Loading SAM model...";
+  showLoading.val = true;
+
+  const ckpt = node.widgets.find((x) => x.name === "ckpt").value;
+  const modelType = ckpt.match(/vit_[lbh]/)?.[0];
+  initModel(modelType).then((res) => {
+    loadingCaption.val = "Computing image embedding...";
+
+    const split = connectedImageFileName.split("/");
+    let id = connectedImageFileName;
+    if (split.length > 1) id = split[1];
+
+    node.widgets.find((x) => x.name === "embedding_id").value = id;
+    embeddingID.val = id;
+
+    api
+      .fetchApi("/sam_model", {
+        method: "POST",
+        body: JSON.stringify({
+          image: connectedImageFileName,
+          isGeneratedImage,
+          embedding_id: id,
+          ckpt,
+          // remote: true,
+        }),
+      })
+      .then(() => {
+        showLoading.val = false;
+        const v = JSON.parse(
+          node.widgets.find((x) => x.name === "image_prompts_json").value
+        );
+
+        if (!Array.isArray(v)) {
+          // this is a multi prompt
+          imagePromptsMulti.val = v;
+          selectedLayer.val = Object.keys(imagePromptsMulti.val)[0];
+          imagePrompts.val = imagePromptsMulti.val[selectedLayer.val];
+        } else {
+          // this is a single prompt
+          selectedLayer.val = "";
+          imagePromptsMulti.val = {};
+          imagePrompts.val = v;
+        }
+        showImageEditor.val = true;
+        const subfolder =
+          isGeneratedImage || split.length === 1 ? "" : split[0];
+        imageUrl.val = api.apiURL(
+          `/view?filename=${encodeURIComponent(connectedImageFileName)}&type=${
+            isGeneratedImage ? "output" : "input"
+          }&subfolder=${subfolder}`
+        );
+        const embeedingUrl = api.apiURL(
+          `/view?filename=${encodeURIComponent(
+            `${id}_${modelType}.npy`
+          )}&type=output&subfolder=`
+        );
+        loadNpyTensor(embeedingUrl).then(async (tensor) => {
+          embeddings.val = tensor;
+          if (enableAutoSegment.val && !segmented.val) await autoSegment();
+          drawSegment(getClicks());
+          updateImagePrompts();
+        });
+        targetNode.val = node;
+      })
+      .catch((err) => {
+        console.log(err);
+        showLoading.val = false;
+      });
   });
-  targetNode.val = node;
 }
 
 /** @typedef {import('../../../web/types/comfy.js').ComfyExtension} ComfyExtension*/
@@ -294,47 +372,19 @@ const ext = {
   getCustomWidgets(app) {
     return {
       SAM_PROMPTS(node, inputName, inputData, app) {
-        const btn = node.addWidget("button", "Edit prompt", "", () => {
-          let connectedImageFileName = getInputWidgetValue(node, 0, "image");
-          if (!connectedImageFileName) {
-            alertDialog.val = {
-              text: "Please connect an image first",
-              time: 3000,
-            };
-            return;
-          }
-
-          loadingCaption.val = "Computing image embedding...";
-          showLoading.val = true;
-
-          const split = connectedImageFileName.split("/");
-          let id = connectedImageFileName;
-          if (split.length > 1) id = split[1];
-
-          node.widgets.find((x) => x.name === "embedding_id").value = id;
-
-          const ckpt = node.widgets.find((x) => x.name === "ckpt").value;
-
-          api
-            .fetchApi("/sam_model", {
-              method: "POST",
-              body: JSON.stringify({
-                image: connectedImageFileName,
-                embedding_id: id,
-                ckpt,
-              }),
-            })
-            .then(() => {
-              showLoading.val = false;
-              showMyImageEditor(node);
-            })
-            .catch((err) => {
-              console.log(err);
-              showLoading.val = false;
-            });
+        const asd = document.createElement("div");
+        Object.assign(asd, {
+          id: "sam",
+          onclick: () => {
+            showMyImageEditor(node);
+          },
         });
-        btn.serialize = false;
+        document.body.append(asd);
 
+        const btn = node.addWidget("button", "Edit prompt", "", () => {
+          showMyImageEditor(node);
+          btn.serialize = false;
+        });
         return {
           widget: btn,
         };
@@ -377,6 +427,24 @@ const ext = {
           widget: btn,
         };
       },
+      GROUP_OPS(node, inputName, inputData, app) {
+        const btn = node.addWidget("button", "Add OBJ", "", () => {
+          node.addInput("BPY_OBJ" + (node.inputs.length + 1), "BPY_OBJ");
+          node.graph.change();
+        });
+        return {
+          widget: btn,
+        };
+      },
+      GROUP_OPS_DELETE(node, inputName, inputData, app) {
+        const btn = node.addWidget("button", "Delete OBJ", "", () => {
+          node.removeInput(node.inputs.length - 1);
+          node.graph.change();
+        });
+        return {
+          widget: btn,
+        };
+      },
     };
   },
 
@@ -393,6 +461,7 @@ const ext = {
 
       node.computeParentGroupResize();
     };
+    injectUIComponentToComfyuimenu();
   },
 
   async setup() {
@@ -405,6 +474,10 @@ const ext = {
     });
 
     api.addEventListener("executed", (evt) => {
+      const images = evt.detail?.output.images;
+      if (images?.length > 0 && images[0].type === "output") {
+        generatedImages[evt.detail.node] = images[0].filename;
+      }
       if (evt.detail?.output.gltfFilename) {
         const viewer = document.getElementById(
           "avatech-viewer-iframe"
@@ -442,7 +515,7 @@ const ext = {
 
     window.addEventListener(
       "keydown",
-      (event) => {
+      async (event) => {
         if (event.key === "Escape") {
           event.preventDefault();
           if (my_modal_3.open) {
@@ -450,6 +523,14 @@ const ext = {
           } else {
             showImageEditor.val = false;
             showEditor.val = false;
+            await uploadSegments();
+            // api.fetchApi("/segments_order", {
+            //   method: "POST",
+            //   body: JSON.stringify({
+            //     name: embeddingID.val,
+            //     order: Object.keys(imagePromptsMulti.val),
+            //   }),
+            // });
           }
         }
       },
@@ -590,20 +671,7 @@ const ext = {
           });
         });
         break;
-      case "SAM_Prompt_Image":
-        nodeData.input.required.sam = ["SAM_PROMPTS"];
-        // nodeData.input.required.upload = ['IMAGEUPLOAD'];
-        // nodeData.input.required.prompts_points = ["IMAGEUPLOAD"];
-        addMenuHandler(nodeType, function (_, options) {
-          options.unshift({
-            content: "Open In Points Editor (Local)",
-            callback: () => {
-              showMyImageEditor(this);
-            },
-          });
-        });
-        break;
-      case "SAM":
+      case "SAM MultiLayer":
         nodeData.input.required.sam = ["SAM_PROMPTS"];
         // nodeData.input.required.upload = ['IMAGEUPLOAD'];
         // nodeData.input.required.prompts_points = ["IMAGEUPLOAD"];
@@ -623,10 +691,189 @@ const ext = {
         nodeData.input.required.obj = ["MESH_GROUP_CONFIG"];
         nodeData.input.required.del_obj = ["MESH_GROUP_DELETE"];
         break;
+      case "GroupOps":
+        nodeData.input.required.obj = ["GROUP_OPS"];
+        nodeData.input.required.del_obj = ["GROUP_OPS_DELETE"];
       default:
         break;
     }
   },
 };
 
+export async function uploadPreview() {
+  if (fileName.val == "")
+    app.ui.dialog.show("Please create your avatar first.");
+  else {
+    const file = await fetch(fileName.val)
+      .then((e) => e.arrayBuffer())
+      .then((e) => new Uint8Array(e));
+    const labData = await fetch("https://labs.avatech.ai/api/share", {
+      method: "GET",
+    }).then((e) => e.json());
+
+    await fetch(labData.url, {
+      method: "PUT",
+      headers: {
+        "x-amz-acl": "public-read",
+        "Content-Type": "model/gltf-binary",
+        "Content-Length": file.length,
+      },
+      body: file,
+    }).catch((error) => console.error(error));
+
+    sharedAvatarLink.val = `https://editor.avatech.ai/viewer?avatarId=${labData?.modelId}`;
+    previewModelId.val = labData.modelId;
+    return labData;
+  }
+}
+
+function injectUIComponentToComfyuimenu() {
+  const menu = document.querySelector(".comfy-menu");
+  const avatarPreview = document.createElement("button");
+  avatarPreview.textContent = "Avatar Preview";
+  avatarPreview.onclick = () => {
+    showPreview.val = !showPreview.val;
+    localStorage.setItem("showPreview", showPreview.val);
+  };
+
+  const apiFormat = document.createElement("button");
+  const a = document.createElement("a");
+  apiFormat.textContent = "Save API Format (Avatech)";
+  apiFormat.onclick = () => {
+    let filename = "workflow_api.json";
+    filename = prompt("Save workflow (API) as:", filename);
+    if (!filename) return;
+    if (!filename.toLowerCase().endsWith(".json")) {
+      filename += ".json";
+    }
+    app.graphToPrompt().then(p=>{
+      console.log('fkfk');
+      let json = JSON.stringify(p.output, null, 2); // convert the data to a JSON string
+      json = json.replace(/"seed": (\d+)/g, `"seed": "SEED"`).replace(/"image": "(?!.*mask.*\.png).*"/g, '"image": "reference_image_avatech"').replace(/"embedding_id": ".*"/g, '"embedding_id": "embedding_id_avatech"');
+      const blob = new Blob([json], {type: "application/json"});
+      const url = URL.createObjectURL(blob);
+      a.href = url;
+      a.download = filename;
+      document.body.appendChild(a);
+      a.click();
+      setTimeout(function () {
+        a.remove();
+        window.URL.revokeObjectURL(url);
+      }, 0);
+    });
+  };
+
+  const dropdown = document.createElement("div");
+  dropdown.textContent = "▼";
+  dropdown.className = "dropdownbtn";
+  dropdown.onclick = (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+
+    LiteGraph.closeAllContextMenus();
+    const menu = new LiteGraph.ContextMenu(
+      [
+        {
+          title: "Create new share link",
+          callback: async () => {
+            shareAvatar.textContent = "Loading...";
+            shareAvatar.append(dropdown);
+
+            await uploadPreview();
+
+            shareLoading.val = false;
+            shareAvatar.textContent = "Share Avatar";
+            shareAvatar.append(dropdown);
+          },
+        },
+        {
+          title: "Update avatar in current share link",
+          callback: async () => {
+            if (!previewModelId.val)
+              app.ui.dialog.show("Please share your avatar first.");
+            else {
+              if (shareLoading.val) return;
+
+              const file = await fetch(fileName.val)
+                .then((e) => e.arrayBuffer())
+                .then((e) => new Uint8Array(e));
+
+              const labData = await fetch(
+                "https://labs.avatech.ai/api/share?id=" + previewModelId.val,
+                {
+                  method: "GET",
+                }
+              ).then((e) => e.json());
+
+              await fetch(labData.url, {
+                method: "PUT",
+                headers: {
+                  "x-amz-acl": "public-read",
+                  "Content-Type": "model/gltf-binary",
+                  "Content-Length": file.length,
+                },
+                body: file,
+              }).catch((error) => console.error(error));
+
+              await fetch(
+                "https://labs.avatech.ai/api/purgecdn?id=" + previewModelId.val,
+                {
+                  method: "GET",
+                }
+              ).catch((error) => console.error(error));
+
+              infoDialog.show(
+                `Preview updated: <a href='https://editor.avatech.ai/viewer?avatarId=${labData.modelId}' target="_blank">https://editor.avatech.ai/viewer?avatarId=` +
+                  labData.modelId +
+                  "</a>\n Remember to hard refresh before checking out the new preview!"
+              );
+
+              shareLoading.val = false;
+              shareAvatar.textContent = "Share Avatar";
+              shareAvatar.append(dropdown);
+            }
+          },
+        },
+      ],
+
+      {
+        event: e,
+        scale: 1.3,
+      },
+      window
+    );
+    menu.root.classList.add("popup");
+  };
+
+  const shareAvatar = document.createElement("button");
+  shareAvatar.textContent = "Share Avatar";
+  shareAvatar.className = "sharebtn";
+  shareAvatar.onclick = async () => {
+    if (shareLoading.val) return;
+
+    if (!previewModelId.val) {
+      shareLoading.val = true;
+      shareAvatar.textContent = "Loading...";
+      shareAvatar.append(dropdown);
+
+      await uploadPreview();
+
+      shareLoading.val = false;
+      shareAvatar.textContent = "Share Avatar";
+      shareAvatar.append(dropdown);
+    } else {
+      infoDialog.show(
+        `Preview avatar url: <a href='https://editor.avatech.ai/viewer?avatarId=${previewModelId.val}' target="_blank">https://editor.avatech.ai/viewer?avatarId=${previewModelId.val}</a>` +
+          `\nChat url: <a href='https://labs.avatech.ai?avatarId=${previewModelId.val}' target="_blank">https://labs.avatech.ai?avatarId=${previewModelId.val}</a>`
+      );
+    }
+  };
+
+  menu.append(avatarPreview);
+  menu.append(shareAvatar);
+  menu.append(apiFormat);
+
+  shareAvatar.append(dropdown);
+}
+
 app.registerExtension(ext);

js/onnx.js

@@ -5,19 +5,16 @@ import { modelData, onnxMaskToImage } from "./onnx_helper.js";
 
 ort.env.wasm.wasmPaths = "https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/";
 
-// Define image, embedding and model paths
-const IMAGE_PATH = "/assets/data/dogs.jpg";
-const IMAGE_EMBEDDING = "/assets/data/dogs_embedding.npy";
-const MODEL_DIR = "http://127.0.0.1:8188/sam_model";
-
 export let model = null;
 
 // Initialize the ONNX model
-export const initModel = async () => {
+export const initModel = async (modelType) => {
   try {
-    if (MODEL_DIR === undefined) return;
-    const URL = MODEL_DIR;
-    model = await ort.InferenceSession.create(URL);
+    if (!model) {
+      model = await ort.InferenceSession.create(
+        `${location.protocol}//${location.host}/sam_model?type=${modelType}`
+      );
+    }
   } catch (e) {
     console.log(e);
   }
@@ -25,14 +22,12 @@ export const initModel = async () => {
 
 export const loadNpyTensor = async (tensorFile, dType = "float32") => {
   let npLoader = new npyjs();
-  console.log('tensorFile', tensorFile);
   const npArray = await npLoader.load(tensorFile);
-  console.log('np array', npArray);
   const tensor = new ort.Tensor(dType, npArray.data, npArray.shape);
   return tensor;
 };
 
-export const runONNX = async (clicks, tensor) => {
+export const runONNX = async (clicks, tensor, box) => {
   // console.log('tensor', tensor);
   try {
     if (
@@ -49,6 +44,7 @@ export const runONNX = async (clicks, tensor) => {
         clicks,
         tensor,
         modelScale: imageSize.val,
+        box,
       });
       if (feeds === undefined) return;
       // Run the SAM ONNX model with the feeds returned from modelData()

js/onnx_helper.js

@@ -4,7 +4,7 @@
 // This source code is licensed under the license found in the
 // LICENSE file in the root directory of this source tree.
 
-const modelData = ({ clicks, tensor, modelScale }) => {
+const modelData = ({ clicks, tensor, modelScale, box }) => {
   const imageEmbedding = tensor;
   let pointCoords;
   let pointLabels;
@@ -18,8 +18,9 @@ const modelData = ({ clicks, tensor, modelScale }) => {
     // If there is no box input, a single padding point with
     // label -1 and coordinates (0.0, 0.0) should be concatenated
     // so initialize the array to support (n + 1) points.
-    pointCoords = new Float32Array(2 * (n + 1));
-    pointLabels = new Float32Array(n + 1);
+    const numPoints = box ? n + 3 : n + 1;
+    pointCoords = new Float32Array(2 * numPoints);
+    pointLabels = new Float32Array(numPoints);
 
     // Add clicks and scale to what SAM expects
     for (let i = 0; i < n; i++) {
@@ -28,15 +29,25 @@ const modelData = ({ clicks, tensor, modelScale }) => {
       pointLabels[i] = clicks[i].clickType;
     }
 
-    // Add in the extra point/label when only clicks and no box
-    // The extra point is at (0, 0) with label -1
-    pointCoords[2 * n] = 0.0;
-    pointCoords[2 * n + 1] = 0.0;
-    pointLabels[n] = -1.0;
+    if (box) {
+      pointCoords[2 * n] = box.x1 * modelScale.samScale;
+      pointCoords[2 * n + 1] = box.y1 * modelScale.samScale;
+      pointLabels[n] = 2;
+
+      pointCoords[2 * n + 2] = box.x2 * modelScale.samScale;
+      pointCoords[2 * n + 3] = box.y2 * modelScale.samScale;
+      pointLabels[n + 1] = 3;
+    } else {
+      // Add in the extra point/label when only clicks and no box
+      // The extra point is at (0, 0) with label -1
+      pointCoords[2 * n] = 0.0;
+      pointCoords[2 * n + 1] = 0.0;
+      pointLabels[n] = -1.0;
+    }
 
     // Create the tensor
-    pointCoordsTensor = new ort.Tensor("float32", pointCoords, [1, n + 1, 2]);
-    pointLabelsTensor = new ort.Tensor("float32", pointLabels, [1, n + 1]);
+    pointCoordsTensor = new ort.Tensor("float32", pointCoords, [1, numPoints, 2]);
+    pointLabelsTensor = new ort.Tensor("float32", pointLabels, [1, numPoints]);
   }
   const imageSizeTensor = new ort.Tensor("float32", [
     modelScale.height,

js/state.js

@@ -9,23 +9,40 @@
  * @typedef {Object} Point
  * @property {number} x - The x coordinate
  * @property {number} y - The y coordinate
- * @property {number} label - The label
+ * @property {>} label - The label
+ *
+ * @typedef {Object} Box
+ * @property {number} x1
+ * @property {number} y1
+ * @property {number} x2
+ * @property {number} y2
  */
 
 import { van } from "./van.js";
 
 export const iframeSrc = van.state("https://editor.avatech.ai?comfyui=true");
 export const showEditor = van.state(false);
-export const previewUrl = van.state("https://editor.avatech.ai/viewer?avatarId=default&hideUI=true&debug=true&width=300&height=300&showAudioControl=true");
+// localStorage.getItem("showPreview") == 'true'
+export const showPreview = van.state(true);
+export const previewUrl = van.state(
+  "https://editor.avatech.ai/viewer?avatarId=default&debug=true&width=350&height=350&hideTrigger=true&voiceSelection=true&hideUI=true"
+);
+export const previewImg = van.state("");
+export const previewImgLoading = van.state(false);
+export const enableAutoSegment = van.state(false);
 // export const previewUrl = van.state("http://localhost:3006/viewer?avatarId=default&hideUI=true&debug=true&width=300&height=300&showAudioControl=true");
 export const isDirty = van.state(false);
-export const fileName = van.state('');
+export const fileName = van.state("");
 export const showImageEditor = van.state(false);
 export const showLoading = van.state(false);
 export const alertDialog = van.state({
   text: "",
   time: 0,
 });
+export const shareLoading = van.state(false);
+export const previewModelId = van.state("");
+
+export const isGenerateFlow = van.state(false);
 
 export const loadingCaption = van.state("");
 export const imageUrl = van.state("");
@@ -35,9 +52,18 @@ export const imageContainerSize = van.state({
   height: 0,
 });
 
+/** @type {State<Box>} */
+export const boxes = van.state();
+
+/** @type {State<Record<string, Box>>} */
+export const boxesMulti = van.state({});
+
 /** @type {State<Point[]>} */
 export const imagePrompts = van.state([]);
 
+export const allImagePrompts = van.state([{}]);
+
+
 /** @type {State<Record<string, Point[]>>} */
 export const imagePromptsMulti = van.state({});
 
@@ -49,3 +75,4 @@ export const targetNode = van.state();
 
 export const imageSize = van.state({ width: 0, height: 0, samScale: 0 });
 export const embeddings = van.state();
+export const embeddingID = van.state("Test");

js/styles.css

@@ -1,3 +1,146 @@
+@import url('https://fonts.googleapis.com/css2?family=Gabarito&display=swap');
 @tailwind base;
 @tailwind components;
-@tailwind utilities;
+@tailwind utilities;
+
+.comfy-normal-modal {
+  display: none; /* Hidden by default */
+  position: fixed; /* Stay in place */
+  z-index: 100; /* Sit on top */
+  padding: 30px 30px 10px 30px;
+  background-color: var(--comfy-menu-bg); /* Modal background */
+  box-shadow: 0 0 20px #888888;
+  border-radius: 10px;
+  top: 50%;
+  left: 50%;
+  max-width: 80vw;
+  max-height: 80vh;
+  transform: translate(-50%, -50%);
+  overflow: hidden;
+  justify-content: center;
+  font-family: monospace;
+  font-size: 15px;
+  color: #ffffff;
+}
+
+.comfy-normal-modal p {
+  overflow: auto;
+  white-space: pre-line; /* This will respect line breaks */
+  margin-bottom: 20px; /* Add some margin between the text and the close button*/
+}
+
+.comfy-normal-modal a {
+  text-decoration-line: underline;
+}
+
+.comfy-normal-modal button {
+  font-size: 20px;
+  color: var(--input-text);
+  background-color: var(--comfy-input-bg);
+  border-radius: 8px;
+  border-color: var(--border-color);
+  border-style: solid;
+  margin-top: 2px;
+}
+
+.comfy-normal-modal button:hover {
+  filter: brightness(1.2);
+  cursor: pointer;
+}
+
+.sharebtn {
+  display: flex;
+  justify-content: flex-end;
+  gap: 0.25rem;
+}
+
+.popup ~ .litecontextmenu {
+  transform: scale(1.3);
+}
+
+.dropdownbtn {
+  font-size: 12px;
+  display: flex;
+  align-items: center;
+  width: 24px;
+  height: 30px;
+  justify-content: center;
+  background: rgba(255, 255, 255, 0.1);
+  border-top-right-radius: 0.375rem;
+  border-bottom-right-radius: 0.375rem;
+}
+
+.dropdownbtn:hover {
+  filter: brightness(1.6);
+  background-color: var(--comfy-menu-bg);
+}
+
+.comfy-menu > button,
+.comfy-menu-btns button,
+.comfy-menu .comfy-list button,
+.comfy-modal button {
+  color: var(--input-text);
+  background-color: var(--comfy-input-bg);
+  border-radius: 8px;
+  border-color: var(--border-color);
+  border-style: solid;
+  margin-top: 2px;
+  border-width: 2px;
+}
+
+.comfy-menu > button:hover,
+.comfy-menu-btns button:hover,
+.comfy-menu .comfy-list button:hover,
+.comfy-modal button:hover,
+.comfy-settings-btn:hover {
+  filter: brightness(1.2);
+  cursor: pointer;
+}
+
+.comfy-list {
+  color: var(--descrip-text);
+  background-color: var(--comfy-menu-bg);
+  margin-bottom: 10px;
+  border-color: var(--border-color);
+  border-style: solid;
+  border-width: 3px;
+}
+
+.comfy-list-items {
+  overflow-y: scroll;
+  max-height: 100px;
+  min-height: 25px;
+  background-color: var(--comfy-input-bg);
+  padding: 5px;
+}
+
+.comfy-list h4 {
+  min-width: 160px;
+  margin: 0;
+  padding: 3px;
+  font-weight: normal;
+}
+
+.comfy-list-items button {
+  font-size: 10px;
+}
+
+.comfy-list-actions {
+  margin: 5px;
+  display: flex;
+  gap: 5px;
+  justify-content: center;
+}
+
+.comfy-list-actions button {
+  font-size: 12px;
+}
+
+img {
+  display: none;
+}
+
+
+img[src] {
+ display: block;
+}

package.json

@@ -11,7 +11,7 @@
   "license": "ISC",
   "devDependencies": {
     "chokidar": "^3.5.3",
-    "daisyui": "^3.7.5",
+    "daisyui": "^4.0.7",
     "tailwindcss": "^3.3.3"
   }
 }

pnpm-lock.yaml

@@ -8,12 +8,9 @@ devDependencies:
   chokidar:
     specifier: ^3.5.3
     version: 3.5.3
-  concurrently:
-    specifier: ^8.2.1
-    version: 8.2.1
   daisyui:
-    specifier: ^3.7.5
-    version: 3.7.5
+    specifier: ^4.0.7
+    version: 4.0.7(postcss@8.4.29)
   tailwindcss:
     specifier: ^3.3.3
     version: 3.3.3
@@ -25,13 +22,6 @@ packages:
     engines: {node: '>=10'}
     dev: true
 
-  /@babel/runtime@7.22.11:
-    resolution: {integrity: sha512-ee7jVNlWN09+KftVOu9n7S8gQzD/Z6hN/I8VBRXW4P1+Xe7kJGXMwu8vds4aGIMHZnNbdpSWCfZZtinytpcAvA==}
-    engines: {node: '>=6.9.0'}
-    dependencies:
-      regenerator-runtime: 0.14.0
-    dev: true
-
   /@jridgewell/gen-mapping@0.3.3:
     resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==}
     engines: {node: '>=6.0.0'}
@@ -83,18 +73,6 @@ packages:
       fastq: 1.15.0
     dev: true
 
-  /ansi-regex@5.0.1:
-    resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
-    engines: {node: '>=8'}
-    dev: true
-
-  /ansi-styles@4.3.0:
-    resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
-    engines: {node: '>=8'}
-    dependencies:
-      color-convert: 2.0.1
-    dev: true
-
   /any-promise@1.3.0:
     resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==}
     dev: true
@@ -139,14 +117,6 @@ packages:
     engines: {node: '>= 6'}
     dev: true
 
-  /chalk@4.1.2:
-    resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
-    engines: {node: '>=10'}
-    dependencies:
-      ansi-styles: 4.3.0
-      supports-color: 7.2.0
-    dev: true
-
   /chokidar@3.5.3:
     resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==}
     engines: {node: '>= 8.10.0'}
@@ -162,30 +132,6 @@ packages:
       fsevents: 2.3.3
     dev: true
 
-  /cliui@8.0.1:
-    resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==}
-    engines: {node: '>=12'}
-    dependencies:
-      string-width: 4.2.3
-      strip-ansi: 6.0.1
-      wrap-ansi: 7.0.0
-    dev: true
-
-  /color-convert@2.0.1:
-    resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
-    engines: {node: '>=7.0.0'}
-    dependencies:
-      color-name: 1.1.4
-    dev: true
-
-  /color-name@1.1.4:
-    resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
-    dev: true
-
-  /colord@2.9.3:
-    resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==}
-    dev: true
-
   /commander@4.1.1:
     resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==}
     engines: {node: '>= 6'}
@@ -195,22 +141,6 @@ packages:
     resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
     dev: true
 
-  /concurrently@8.2.1:
-    resolution: {integrity: sha512-nVraf3aXOpIcNud5pB9M82p1tynmZkrSGQ1p6X/VY8cJ+2LMVqAgXsJxYYefACSHbTYlm92O1xuhdGTjwoEvbQ==}
-    engines: {node: ^14.13.0 || >=16.0.0}
-    hasBin: true
-    dependencies:
-      chalk: 4.1.2
-      date-fns: 2.30.0
-      lodash: 4.17.21
-      rxjs: 7.8.1
-      shell-quote: 1.8.1
-      spawn-command: 0.0.2
-      supports-color: 8.1.1
-      tree-kill: 1.2.2
-      yargs: 17.7.2
-    dev: true
-
   /css-selector-tokenizer@0.8.0:
     resolution: {integrity: sha512-Jd6Ig3/pe62/qe5SBPTN8h8LeUg/pT4lLgtavPf7updwwHpvFzxvOQBHYj2LZDMjUnBzgvIUSjRcf6oT5HzHFg==}
     dependencies:
@@ -224,24 +154,21 @@ packages:
     hasBin: true
     dev: true
 
-  /daisyui@3.7.5:
-    resolution: {integrity: sha512-udhiBJYVvcPGXa+mL5IElke6EdddKecjbbz6m43+9IVqzK8GBwetg092Edclo42TkNboLD9nzodeesJygqZt2A==}
-    engines: {node: '>=16.9.0'}
-    dependencies:
-      colord: 2.9.3
-      css-selector-tokenizer: 0.8.0
-      postcss: 8.4.29
-      postcss-js: 4.0.1(postcss@8.4.29)
-      tailwindcss: 3.3.3
-    transitivePeerDependencies:
-      - ts-node
+  /culori@3.2.0:
+    resolution: {integrity: sha512-HIEbTSP7vs1mPq/2P9In6QyFE0Tkpevh0k9a+FkjhD+cwsYm9WRSbn4uMdW9O0yXlNYC3ppxL3gWWPOcvEl57w==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
     dev: true
 
-  /date-fns@2.30.0:
-    resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==}
-    engines: {node: '>=0.11'}
+  /daisyui@4.0.7(postcss@8.4.29):
+    resolution: {integrity: sha512-D84DnNDZKcamwNsxCMrwYaddyz5kC6VO6oe30nM1x67GzCAfarfd3Ar1rpLGXCIqSsEoNZUHO8EcXvX93W2ZkA==}
+    engines: {node: '>=16.9.0'}
     dependencies:
-      '@babel/runtime': 7.22.11
+      css-selector-tokenizer: 0.8.0
+      culori: 3.2.0
+      picocolors: 1.0.0
+      postcss-js: 4.0.1(postcss@8.4.29)
+    transitivePeerDependencies:
+      - postcss
     dev: true
 
   /didyoumean@1.2.2:
@@ -252,15 +179,6 @@ packages:
     resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==}
     dev: true
 
-  /emoji-regex@8.0.0:
-    resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
-    dev: true
-
-  /escalade@3.1.1:
-    resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==}
-    engines: {node: '>=6'}
-    dev: true
-
   /fast-glob@3.3.1:
     resolution: {integrity: sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==}
     engines: {node: '>=8.6.0'}
@@ -305,11 +223,6 @@ packages:
     resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==}
     dev: true
 
-  /get-caller-file@2.0.5:
-    resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
-    engines: {node: 6.* || 8.* || >= 10.*}
-    dev: true
-
   /glob-parent@5.1.2:
     resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
     engines: {node: '>= 6'}
@@ -335,11 +248,6 @@ packages:
       path-is-absolute: 1.0.1
     dev: true
 
-  /has-flag@4.0.0:
-    resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
-    engines: {node: '>=8'}
-    dev: true
-
   /has@1.0.3:
     resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==}
     engines: {node: '>= 0.4.0'}
@@ -376,11 +284,6 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: true
 
-  /is-fullwidth-code-point@3.0.0:
-    resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
-    engines: {node: '>=8'}
-    dev: true
-
   /is-glob@4.0.3:
     resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
     engines: {node: '>=0.10.0'}
@@ -407,10 +310,6 @@ packages:
     resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
     dev: true
 
-  /lodash@4.17.21:
-    resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
-    dev: true
-
   /merge2@1.4.1:
     resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}
     engines: {node: '>= 8'}
@@ -580,15 +479,6 @@ packages:
       picomatch: 2.3.1
     dev: true
 
-  /regenerator-runtime@0.14.0:
-    resolution: {integrity: sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==}
-    dev: true
-
-  /require-directory@2.1.1:
-    resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
-    engines: {node: '>=0.10.0'}
-    dev: true
-
   /resolve@1.22.4:
     resolution: {integrity: sha512-PXNdCiPqDqeUou+w1C2eTQbNfxKSuMxqTCuvlmmMsk1NWHL5fRrhY6Pl0qEYYc6+QqGClco1Qj8XnjPego4wfg==}
     hasBin: true
@@ -609,41 +499,11 @@ packages:
       queue-microtask: 1.2.3
     dev: true
 
-  /rxjs@7.8.1:
-    resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==}
-    dependencies:
-      tslib: 2.6.2
-    dev: true
-
-  /shell-quote@1.8.1:
-    resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==}
-    dev: true
-
   /source-map-js@1.0.2:
     resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==}
     engines: {node: '>=0.10.0'}
     dev: true
 
-  /spawn-command@0.0.2:
-    resolution: {integrity: sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==}
-    dev: true
-
-  /string-width@4.2.3:
-    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
-    engines: {node: '>=8'}
-    dependencies:
-      emoji-regex: 8.0.0
-      is-fullwidth-code-point: 3.0.0
-      strip-ansi: 6.0.1
-    dev: true
-
-  /strip-ansi@6.0.1:
-    resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
-    engines: {node: '>=8'}
-    dependencies:
-      ansi-regex: 5.0.1
-    dev: true
-
   /sucrase@3.34.0:
     resolution: {integrity: sha512-70/LQEZ07TEcxiU2dz51FKaE6hCTWC6vr7FOk3Gr0U60C3shtAN+H+BFr9XlYe5xqf3RA8nrc+VIwzCfnxuXJw==}
     engines: {node: '>=8'}
@@ -658,20 +518,6 @@ packages:
       ts-interface-checker: 0.1.13
     dev: true
 
-  /supports-color@7.2.0:
-    resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
-    engines: {node: '>=8'}
-    dependencies:
-      has-flag: 4.0.0
-    dev: true
-
-  /supports-color@8.1.1:
-    resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==}
-    engines: {node: '>=10'}
-    dependencies:
-      has-flag: 4.0.0
-    dev: true
-
   /supports-preserve-symlinks-flag@1.0.0:
     resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
     engines: {node: '>= 0.4'}
@@ -728,60 +574,19 @@ packages:
       is-number: 7.0.0
     dev: true
 
-  /tree-kill@1.2.2:
-    resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==}
-    hasBin: true
-    dev: true
-
   /ts-interface-checker@0.1.13:
     resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==}
     dev: true
 
-  /tslib@2.6.2:
-    resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==}
-    dev: true
-
   /util-deprecate@1.0.2:
     resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
     dev: true
 
-  /wrap-ansi@7.0.0:
-    resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
-    engines: {node: '>=10'}
-    dependencies:
-      ansi-styles: 4.3.0
-      string-width: 4.2.3
-      strip-ansi: 6.0.1
-    dev: true
-
   /wrappy@1.0.2:
     resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
     dev: true
 
-  /y18n@5.0.8:
-    resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
-    engines: {node: '>=10'}
-    dev: true
-
   /yaml@2.3.2:
     resolution: {integrity: sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==}
     engines: {node: '>= 14'}
     dev: true
-
-  /yargs-parser@21.1.1:
-    resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==}
-    engines: {node: '>=12'}
-    dev: true
-
-  /yargs@17.7.2:
-    resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==}
-    engines: {node: '>=12'}
-    dependencies:
-      cliui: 8.0.1
-      escalade: 3.1.1
-      get-caller-file: 2.0.5
-      require-directory: 2.1.1
-      string-width: 4.2.3
-      y18n: 5.0.8
-      yargs-parser: 21.1.1
-    dev: true

requirements.txt

@@ -3,7 +3,9 @@ numpy
 opencv-python
 opencv-contrib-python
 einops
-bpy
+bpy==3.6.0
 segment-anything
 tqdm
+python-dotenv
+mediapipe
 # -e git+https://github.com/facebookresearch/segment-anything.git#egg=segment_anything

routes.py

@@ -1,6 +1,8 @@
 from aiohttp import web
 from segment_anything import sam_model_registry, SamPredictor
 from PIL import Image, ImageOps
+from dotenv import load_dotenv
+from blender.mesh_utils import upload_avatar_file
 import os
 import requests
 import folder_paths
@@ -8,6 +10,15 @@ import json
 import numpy as np
 import server
 import re
+import base64
+from PIL import Image
+import io
+import time
+import execution
+import random
+
+load_dotenv()
+
 
 # For speeding up ONNX model, see https://github.com/facebookresearch/segment-anything/tree/main/demo#onnx-multithreading-with-sharedarraybuffer
 def inject_headers(original_handler):
@@ -32,29 +43,35 @@ for item in server.PromptServer.instance.routes._items:
     routes.append(item)
 server.PromptServer.instance.routes._items = routes
 
+
 @server.PromptServer.instance.routes.get("/avatar-graph-comfyui/tw-styles.css")
 async def get_web_styles(request):
     filename = os.path.join(os.path.dirname(__file__), "js/tw-styles.css")
     return web.FileResponse(filename)
 
+
 @server.PromptServer.instance.routes.get("/sam_model")
 async def get_sam_model(request):
-    filename = os.path.join(folder_paths.base_path, "web/models/sam.onnx")
-    # print(filename)
+    model_type = request.rel_url.query.get("type", "vit_h")
+    filename = os.path.join(folder_paths.base_path, f"web/models/sam_{model_type}.onnx")
     if not os.path.isfile(filename):
         os.makedirs(os.path.dirname(filename), exist_ok=True)
+        print(f"Downloading ONNX model to {filename}")
         response = requests.get(
-            "https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/models/sam.onnx"
+            f"https://avatech-avatar-dev1.nyc3.cdn.digitaloceanspaces.com/models/sam_{model_type}.onnx"
         )
         response.raise_for_status()
         with open(filename, "wb") as f:
             f.write(response.content)
-        print(f"ONNX model downloaded: {filename}")
+        print(f"ONNX model downloaded")
     return web.FileResponse(filename)
 
 
-def load_image(image):
-    image_path = folder_paths.get_annotated_filepath(image)
+def load_image(image, is_generated_image):
+    if is_generated_image:
+        image_path = f"{folder_paths.get_output_directory()}/{image}"
+    else:
+        image_path = folder_paths.get_annotated_filepath(image)
     i = Image.open(image_path)
     i = ImageOps.exif_transpose(i)
     image = i.convert("RGB")
@@ -65,26 +82,303 @@ def load_image(image):
 @server.PromptServer.instance.routes.post("/sam_model")
 async def post_sam_model(request):
     post = await request.json()
+    is_generated_image = post.get("isGeneratedImage")
     emb_id = post.get("embedding_id")
-    emb_filename = f"{folder_paths.get_output_directory()}/{emb_id}.npy"
+    ckpt = post.get("ckpt")
+    ckpt = folder_paths.get_full_path("sams", ckpt)
+    remote = post.get("remote")
+    model_type = re.findall(r"vit_[lbh]", ckpt)[0]
+    emb_filename = f"{folder_paths.get_output_directory()}/{emb_id}_{model_type}.npy"
+    output_json_filename = (
+        f"{folder_paths.get_output_directory()}/{emb_id}_{model_type}.json"
+    )
     if not os.path.exists(emb_filename):
-        image = load_image(post.get("image"))
-        ckpt = post.get("ckpt")
-        model_type = re.findall(r'vit_[lbh]', ckpt)[0]
-        ckpt = folder_paths.get_full_path("sams", ckpt)
-        sam = sam_model_registry[model_type](checkpoint=ckpt)
-        predictor = SamPredictor(sam)
-
-        image_np = (image * 255).astype(np.uint8)
-        predictor.set_image(image_np)
-        emb = predictor.get_image_embedding().cpu().numpy()
-        np.save(emb_filename, emb)
-        with open(f"{folder_paths.get_output_directory()}/{emb_id}.json", "w") as f:
-            json.dump(
-                {
-                    "input_size": predictor.input_size,
-                    "original_size": predictor.original_size,
+        image = load_image(post.get("image"), is_generated_image)
+        if remote:
+            # Run embed in remote server
+            image = Image.fromarray((image * 255).astype(np.uint8))
+            buffered = io.BytesIO()
+            image.save(buffered, format="PNG")
+            image = base64.b64encode(buffered.getvalue()).decode()
+            res = requests.post(
+                "https://avatechgg--sam-embed.modal.run",
+                headers={
+                    "Content-type": "application/json",
+                    "Accept": "application/json",
                 },
-                f,
+                data=json.dumps(
+                    {
+                        "image": image,
+                    }
+                ),
+            ).json()
+            emb, input_size, original_size = (
+                res["emb"],
+                res["input_size"],
+                res["original_size"],
             )
+            emb = np.array(emb).astype(np.float32)
+            np.save(emb_filename, emb)
+            with open(output_json_filename, "w") as f:
+                data = {
+                    "input_size": input_size,
+                    "original_size": original_size,
+                }
+                json.dump(data, f)
+        else:
+            sam = sam_model_registry[model_type](checkpoint=ckpt)
+            predictor = SamPredictor(sam)
+
+            image_np = (image * 255).astype(np.uint8)
+            predictor.set_image(image_np)
+            emb = predictor.get_image_embedding().cpu().numpy()
+            np.save(emb_filename, emb)
+            with open(output_json_filename, "w") as f:
+                json.dump(
+                    {
+                        "input_size": predictor.input_size,
+                        "original_size": predictor.original_size,
+                    },
+                    f,
+                )
+    print("Finished embedding")
     return web.json_response({})
+
+
+def save_image(image, save_name=None):
+    input_folder = folder_paths.get_input_directory()
+    name, extension = os.path.splitext(image.filename)
+
+    if save_name == None:
+        save_name = f"{name}{extension}"
+        i = 1
+        while os.path.exists(f"{input_folder}/{save_name}"):
+            save_name = f"{name}_{i}{extension}"
+            i += 1
+
+    with open(f"{input_folder}/{save_name}", "wb") as f:
+        f.write(image.file.read())
+
+    return save_name
+
+
+def post_prompt(json_data):
+    prompt_server = server.PromptServer.instance
+    json_data = prompt_server.trigger_on_prompt(json_data)
+
+    if "number" in json_data:
+        number = float(json_data["number"])
+    else:
+        number = prompt_server.number
+        if "front" in json_data:
+            if json_data["front"]:
+                number = -number
+
+        prompt_server.number += 1
+
+    if "prompt" in json_data:
+        prompt = json_data["prompt"]
+        valid = execution.validate_prompt(prompt)
+        extra_data = {}
+        if "extra_data" in json_data:
+            extra_data = json_data["extra_data"]
+
+        if "client_id" in json_data:
+            extra_data["client_id"] = json_data["client_id"]
+        if valid[0]:
+            prompt_id = str(uuid.uuid4())
+            outputs_to_execute = valid[2]
+            prompt_server.prompt_queue.put(
+                (number, prompt_id, prompt, extra_data, outputs_to_execute)
+            )
+            response = {
+                "prompt_id": prompt_id,
+                "number": number,
+                "node_errors": valid[3],
+            }
+            return web.json_response(response)
+        else:
+            print("invalid prompt:", valid[1])
+            return web.json_response(
+                {"error": valid[1], "node_errors": valid[3]}, status=400
+            )
+    else:
+        return web.json_response({"error": "no prompt", "node_errors": []}, status=400)
+
+
+def randomSeed(num_digits=15):
+    range_start = 10 ** (num_digits - 1)
+    range_end = (10**num_digits) - 1
+    return random.randint(range_start, range_end)
+
+
+def load_workflow(workflow_name):
+    with open(
+        os.path.join(
+            os.path.dirname(__file__),
+            f"workflow_templates/api/{workflow_name}.json",
+        )
+    ) as f:
+        return "\n".join(f.readlines())
+
+
+
+@server.PromptServer.instance.routes.post("/avatar_generation")
+async def post_prompt_block(request):
+    prompt_server = server.PromptServer.instance
+    post = await request.post()
+    uploaded_workflow = post.get("workflow")
+    workflow_name = post.get("workflow_name")
+    if uploaded_workflow is not None:
+        workflow = uploaded_workflow
+    elif workflow_name is not None:
+        workflow = load_workflow(workflow_name)
+
+    ref_image = post.get("ref_image")
+    base_image = post.get("base_image")
+    if ref_image is not None:
+        image_path = save_image(ref_image)
+        image_name, image_ext = os.path.splitext(image_path)
+        workflow = workflow.replace("reference_image_avatech", image_path)
+    elif base_image is not None:
+        image_path = save_image(base_image)
+        image_name, image_ext = os.path.splitext(image_path)
+        workflow = workflow.replace("base_image", image_path)
+        workflow = workflow.replace("reference_image_avatech", image_path)  # TMP
+
+    for key, value in post.items():
+        if key.startswith("mask_"):
+            mask_name = image_name + "_" + key.replace("mask_", "") + image_ext
+            mask_path = save_image(value, save_name=mask_name)
+            workflow = workflow.replace(f'"{key}"', f'"{mask_path}"')
+
+    workflow = workflow.replace("embedding_id_avatech", image_path)
+    workflow = workflow.replace("SEED", str(randomSeed()))
+    api_prompt = json.loads(workflow)
+
+    # skip generation part if base_image is provided
+    if base_image is not None:
+        for value in api_prompt.values():
+            if (
+                value["class_type"] == "LoadImageFromRequest"
+                and value["inputs"]["name"] == image_path
+            ):
+                del value["inputs"]["image"]
+            elif (
+                value["class_type"] == "PreviewImage"
+                or value["class_type"] == "SaveImage"
+            ):
+                value["inputs"] = {}
+
+    res = post_prompt({"prompt": api_prompt})
+    prompt_id = json.loads(res.text)["prompt_id"]
+    while True:
+        history = prompt_server.prompt_queue.get_history(prompt_id=prompt_id)
+        if history:
+            # file = get_avatar_file(history[prompt_id]["outputs"])
+            # return web.Response(body=file)
+            outputs = history[prompt_id]["outputs"]
+            for node_id, output in outputs.items():
+                if "gltfFilename" in output:
+                    modelId = upload_avatar_file(output)
+                    print("model id", modelId)
+                    return web.json_response({"id": modelId}, status=200)
+        time.sleep(0.5)
+
+
+# @server.PromptServer.instance.routes.get("/get_default_workflow")
+# async def get_default_workflow(request):
+#     # json_link = "https://cdn.discordapp.com/attachments/1119102674437156984/1172255632586448987/workflow_boy_2_1.json?ex=655fa722&is=654d3222&hm=463fa6a3c6ea60f7471196ff45382c729d3b856e86282f905d37a0398711860e&" # YP workflow
+#     # json_link = "https://cdn.discordapp.com/attachments/729003657483518063/1172504658812608572/workflow_15.json?ex=65608f0e&is=654e1a0e&hm=f707d887b9294c1e9b26e54856b1e516d1725a1b25d044b46229cea6e5c804a1&" # Benny workflow
+#     # json_link = 'https://cdn.discordapp.com/attachments/1110859802701221898/1173536418337914970/newstyle.json?ex=65644ff5&is=6551daf5&hm=f129838fae10197351bd27c69c7ff5eb4edf2c7d6ed74e6db8b55ddaa3c77dee&' # Deepwoo workflow
+#     json_link = 'https://cdn.discordapp.com/attachments/729003657483518063/1174045115757633596/girl1114.json?ex=656629b8&is=6553b4b8&hm=df3d7798b887e2b3b6b06ea438f1bc4ba041dd0f9daf54ea48101845ec7f4243&'
+#     response = requests.get(json_link)
+#     response.raise_for_status()
+#     return web.json_response(response.json())
+
+
+@server.PromptServer.instance.routes.get("/get_workflow")
+async def get_workflow(request):
+    name = request.rel_url.query.get("name", "default")
+    # if name == "default":
+    #     json_link = 'https://cdn.discordapp.com/attachments/729003657483518063/1174045115757633596/girl1114.json?ex=656629b8&is=6553b4b8&hm=df3d7798b887e2b3b6b06ea438f1bc4ba041dd0f9daf54ea48101845ec7f4243&'
+    #     response = requests.get(json_link)
+    #     response.raise_for_status()
+    #     workflow = response.json()
+    # else:
+    if name == "default":
+        name = "Auto_segment_workflow"
+
+    workflows_path = os.path.join(os.path.dirname(__file__), "workflow_templates")
+    workflow = json.load(open(f"{workflows_path}/{name}.json"))
+    return web.json_response(workflow)
+
+
+@server.PromptServer.instance.routes.post("/segments")
+async def post_segments(request):
+    post = await request.json()
+    name = post.get("name")
+    segments = post.get("segments")
+    output_dir = os.path.join(folder_paths.base_path, f"output/segments_{name}")
+    os.makedirs(output_dir, exist_ok=True)
+    for key, value in segments.items():
+        filename = os.path.join(output_dir, f"{key}.png")
+        with open(filename, "wb") as f:
+            f.write(base64.b64decode(value.split(",")[1]))
+
+    order = list(segments.keys())
+    with open(os.path.join(output_dir, "order.json"), "w") as f:
+        json.dump(order, f)
+    return web.json_response({})
+
+
+# @server.PromptServer.instance.routes.post("/segments_order")
+# async def post_segments(request):
+#     post = await request.json()
+#     name = post.get("name")
+#     order = post.get("order")
+#     output_dir = os.path.join(folder_paths.base_path, f"output/{name}")
+#     os.makedirs(output_dir, exist_ok=True)
+#     with open(os.path.join(output_dir, "order.json") , "w") as f:
+#         json.dump(order, f)
+#     return web.json_response({})
+
+
+@server.PromptServer.instance.routes.get("/get_webhook")
+async def get_webhook(request):
+    url = os.getenv("DISCORD_WEBHOOK_URL")
+    return web.json_response(url)
+
+
+import uuid
+
+
+@server.PromptServer.instance.routes.post("/create_avatar_from_image")
+async def post_input_file(request):
+    post = await request.read()
+
+    # Doesn't seems working when file isnt png / or nothing is uploaded
+    if not post:
+        raise web.HTTPBadRequest(reason="No image data received")
+
+    try:
+        queue_id = uuid.uuid4()
+
+        output_dir = os.path.join(
+            folder_paths.base_path, "input", "create_avatar_endpoint"
+        )
+        os.makedirs(output_dir, exist_ok=True)
+
+        filename = os.path.join(output_dir, str(queue_id) + ".png")
+        with open(filename, "wb") as f:
+            f.write(post)
+
+        return web.json_response(
+            {
+                "redirect_url": "https://ai-assistant.avatech.ai?queue-id="
+                + str(queue_id)
+            }
+        )
+    except Exception as e:
+        print(e)
+        return web.json_response({"error": e})

sam/load_image_from_request.py

@@ -0,0 +1,43 @@
+import folder_paths
+from PIL import Image, ImageOps
+import numpy as np
+import torch
+
+class LoadImageFromRequest:
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "name": (
+                    "STRING",
+                    {"multiline": False, "default": "face.png"},
+                ),
+            },
+            "optional": {
+                "image": ("IMAGE",),
+            }
+        }
+
+    RETURN_TYPES = ("IMAGE",)
+    RETURN_NAMES = ("image",)
+
+    FUNCTION = "run"
+
+    CATEGORY = "image"
+
+    def run(self, name, image=None):
+        try:
+            image_path = folder_paths.get_annotated_filepath(name)
+            image = Image.open(image_path)
+            image = ImageOps.exif_transpose(image)
+            # image = image.convert("RGB")
+            image = np.array(image).astype(np.float32) / 255.0
+            image = torch.from_numpy(image)[None,]
+            return [image]
+        except:
+            return [image]
+
+
+NODE_CLASS_MAPPINGS = {"LoadImageFromRequest": LoadImageFromRequest}
+
+NODE_DISPLAY_NAME_MAPPINGS = {"LoadImageFromRequest": "Load Image From Request"}

sam/sam.py

@@ -1,108 +0,0 @@
-import folder_paths
-import os
-import numpy as np
-import torch
-import re
-from segment_anything import sam_model_registry, SamPredictor
-from einops import rearrange, repeat
-
-
-global_predictor = None
-
-class SAM:
-    def __init__(self):
-        self.predictor = None
-        self.output_dir = folder_paths.get_output_directory()
-
-    @classmethod
-    def INPUT_TYPES(s):
-        input_dir = folder_paths.get_input_directory()
-        files = [
-            f
-            for f in os.listdir(input_dir)
-            if os.path.isfile(os.path.join(input_dir, f))
-        ]
-        return {
-            "required": {
-                "image": ("IMAGE",),
-                "ckpt": (folder_paths.get_filename_list("sams"),),
-                "embedding_id": (
-                    "STRING",
-                    {"multiline": False, "default": "embedding"},
-                ),
-                # "image": (sorted(files), ),
-                "image_prompts_json": ("STRING", {"multiline": False, "default": "[]"}),
-            },
-        }
-
-    CATEGORY = "image"
-
-    RETURN_TYPES = ("SAM_PROMPT",)
-    FUNCTION = "load_image"
-
-    def load_image(self, image, ckpt, embedding_id, image_prompts_json):
-        import json
-
-        global global_predictor
-
-        if global_predictor is None:
-            ckpt = folder_paths.get_full_path("sams", ckpt)
-            model_type = re.findall(r'vit_[lbh]', ckpt)[0]
-            sam = sam_model_registry[model_type](checkpoint=ckpt)
-            predictor = SamPredictor(sam)
-            global_predictor = predictor
-        
-        predictor = global_predictor
-        
-        emb_filename = f"{self.output_dir}/{embedding_id}.npy"
-        if not os.path.exists(emb_filename):
-            image_np = (image[0].numpy() * 255).astype(np.uint8)
-            predictor.set_image(image_np)
-            emb = predictor.get_image_embedding().cpu().numpy()
-            np.save(emb_filename, emb)
-
-            with open(f"{self.output_dir}/{embedding_id}.json", "w") as f:
-                data = {
-                    "input_size": predictor.input_size,
-                    "original_size": predictor.original_size,
-                }
-                json.dump(data, f)
-        else:
-            emb = np.load(emb_filename)
-
-            with open(f"{self.output_dir}/{embedding_id}.json") as f:
-                data = json.load(f)
-                predictor.input_size = data["input_size"]
-                predictor.features = torch.from_numpy(emb)
-                predictor.is_image_set = True
-                predictor.original_size = data["original_size"]
-
-        image_prompts = json.loads(image_prompts_json)
-
-        result = [image_prompts]
-
-        if isinstance(image_prompts, list):
-            pass
-        elif all(isinstance(item, list) for item in image_prompts.values()):
-            for item in image_prompts.values():
-                if (len(item) == 0):
-                    h, w, c = image[0].shape
-                    result.append(torch.zeros(1, h, w, c))
-                    continue
-                point_coords = np.array([[p['x'], p['y']] for p in item])
-                point_labels = np.array([p['label'] for p in item])
-
-                masks, _, _ = predictor.predict(
-                    point_coords=point_coords,
-                    point_labels=point_labels,
-                )
-                masks = torch.from_numpy(masks)
-                masks = rearrange(masks[0], 'h w -> 1 h w')
-                out_image = repeat(masks, '1 h w -> 1 h w c', c=3) * image
-                result.append(out_image)
-        return result
-
-
-NODE_CLASS_MAPPINGS = {"SAM": SAM}
-
-NODE_DISPLAY_NAME_MAPPINGS = {"SAM": "Segmentation (SAM)"}

sam/sam_multilayer.py

@@ -0,0 +1,372 @@
+import folder_paths
+import os
+import numpy as np
+import torch
+import re
+import json
+from segment_anything import sam_model_registry, SamPredictor
+from einops import rearrange, repeat
+from PIL import Image
+import mediapipe as mp
+from math import sqrt
+
+BaseOptions = mp.tasks.BaseOptions
+FaceLandmarker = mp.tasks.vision.FaceLandmarker
+FaceLandmarkerOptions = mp.tasks.vision.FaceLandmarkerOptions
+PoseLandmarker = mp.tasks.vision.PoseLandmarker
+PoseLandmarkerOptions = mp.tasks.vision.PoseLandmarkerOptions
+VisionRunningMode = mp.tasks.vision.RunningMode
+
+global_predictor = None
+face_landmarker = None
+pose_landmarker = None
+
+# For auto-segmentation
+layerMapping = {
+    "L_eye": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 0,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 0,
+        "positiveScale": 0,
+        "negativeScale": 0.5,
+        "indices": mp.solutions.face_mesh.FACEMESH_LEFT_EYE,
+    },
+    "R_eye": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 0,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 0,
+        "positiveScale": 0,
+        "negativeScale": 0.5,
+        "indices": mp.solutions.face_mesh.FACEMESH_RIGHT_EYE,
+    },
+    "L_iris": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 0,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 0,
+        "positiveScale": -0.2,
+        "negativeScale": 0.5,
+        "indices": mp.solutions.face_mesh.FACEMESH_LEFT_IRIS,
+    },
+    "R_iris": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 0,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 0,
+        "positiveScale": -0.2,
+        "negativeScale": 0.5,
+        "indices": mp.solutions.face_mesh.FACEMESH_RIGHT_IRIS,
+    },
+    "face": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 40,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 60,
+        "positiveScale": 0.2,
+        "negativeScale": 0.6,
+        "indices": mp.solutions.face_mesh.FACEMESH_FACE_OVAL,
+    },
+    "mouth": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 0,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 0,
+        "positiveScale": -0.3,
+        "negativeScale": 0.3,
+        # https://stackoverflow.com/questions/66649492/how-to-get-specific-landmark-of-face-like-lips-or-eyes-using-tensorflow-js-face
+        "indices": [[x, x] for x in [61, 37, 270, 91, 314]],
+    },
+    "mouth_in": {
+        "useMiddle": False,
+        "positiveOffsetX": 0,
+        "positiveOffsetY": 0,
+        "negativeOffsetX": 0,
+        "negativeOffsetY": 0,
+        "positiveScale": -0.5,
+        "negativeScale": 0.5,
+        # https://stackoverflow.com/questions/66649492/how-to-get-specific-landmark-of-face-like-lips-or-eyes-using-tensorflow-js-face
+        "indices": [[x, x] for x in [310, 88]],
+    },
+}
+
+
+class SAMMultiLayer:
+    def __init__(self):
+        self.predictor = None
+        self.output_dir = folder_paths.get_output_directory()
+
+    @classmethod
+    def INPUT_TYPES(s):
+        return {
+            "required": {
+                "image": ("IMAGE",),
+                "ckpt": (folder_paths.get_filename_list("sams"),),
+                "embedding_id": (
+                    "STRING",
+                    {"multiline": False, "default": "embedding"},
+                ),
+                "image_prompts_json": ("STRING", {"multiline": False, "default": "[]"}),
+            },
+        }
+
+    CATEGORY = "image"
+
+    RETURN_TYPES = ("SAM_PROMPT",)
+    FUNCTION = "load_image"
+
+    def load_models(self, ckpt, model_type):
+        global global_predictor, face_landmarker, pose_landmarker
+
+        ckpt = folder_paths.get_full_path("sams", ckpt)
+        sam = sam_model_registry[model_type](checkpoint=ckpt)  # .to("cuda")
+        global_predictor = SamPredictor(sam)
+
+        face_landmarker_model_path = os.path.join(
+            os.path.dirname(__file__), "../mediapipe_models/face_landmarker.task"
+        )
+        face_landmarker_options = FaceLandmarkerOptions(
+            base_options=BaseOptions(model_asset_path=face_landmarker_model_path),
+            running_mode=VisionRunningMode.IMAGE,
+        )
+        face_landmarker = FaceLandmarker.create_from_options(face_landmarker_options)
+
+        pose_landmarker_model_path = os.path.join(
+            os.path.dirname(__file__), "../mediapipe_models/pose_landmarker_full.task"
+        )
+        pose_landmarker_options = PoseLandmarkerOptions(
+            base_options=BaseOptions(model_asset_path=pose_landmarker_model_path),
+            running_mode=VisionRunningMode.IMAGE,
+        )
+        pose_landmarker = PoseLandmarker.create_from_options(pose_landmarker_options)
+        return global_predictor, face_landmarker, pose_landmarker
+
+    def auto_segment(self, image, face_landmarks, pose_landmarks):
+        H, W, C = image.shape
+        imagePromptsMulti = {}
+        boxesMulti = {}
+
+        for key, value in layerMapping.items():
+            positivePoints = []
+            middlePoints = []
+            negativePoints = []
+
+            for index in value["indices"]:
+                start, end = index
+                startPoint = face_landmarks[start]
+
+                startX = startPoint.x * W
+                startY = startPoint.y * H
+
+                if len(middlePoints) == 0:
+                    middlePoints.append({"x": startX, "y": startY, "label": 1})
+                else:
+                    middlePoints[0]["x"] += startX
+                    middlePoints[0]["y"] += startY
+
+                positivePoints.append({"x": startX, "y": startY, "label": 1})
+
+            len_indices = len(value["indices"])
+            middlePoints[0]["x"] /= len_indices
+            middlePoints[0]["y"] /= len_indices
+
+            if value["useMiddle"]:
+                imagePromptsMulti[key] = middlePoints
+            else:
+                for i, index in enumerate(value["indices"]):
+                    start, end = index
+                    startPoint = face_landmarks[start]
+
+                    startX = startPoint.x * W
+                    startY = startPoint.y * H
+
+                    middlePoint = middlePoints[0]
+                    directionVector = {
+                        "x": middlePoint["x"] - startX,
+                        "y": middlePoint["y"] - startY,
+                    }
+                    directionVectorLength = sqrt(
+                        directionVector["x"] * directionVector["x"]
+                        + directionVector["y"] * directionVector["y"]
+                    )
+
+                    if value["negativeScale"] != 0:
+                        negativePointDistance = (
+                            value["negativeScale"] * directionVectorLength
+                        )
+                        negativePoint = {
+                            "x": startX
+                            - (negativePointDistance * directionVector["x"])
+                            / directionVectorLength
+                            - value["negativeOffsetX"],
+                            "y": startY
+                            - (negativePointDistance * directionVector["y"])
+                            / directionVectorLength
+                            - value["negativeOffsetY"],
+                            "label": 0,
+                        }
+                        negativePoints.append(negativePoint)
+
+                    positivePointDistance = (
+                        value["positiveScale"] * directionVectorLength
+                    )
+                    positivePoints[i] = {
+                        "x": positivePoints[i]["x"]
+                        - (positivePointDistance * directionVector["x"])
+                        / directionVectorLength
+                        - value["positiveOffsetX"],
+                        "y": positivePoints[i]["y"]
+                        - (positivePointDistance * directionVector["y"])
+                        / directionVectorLength
+                        - value["positiveOffsetY"],
+                        "label": 1,
+                    }
+
+                imagePromptsMulti[key] = positivePoints + negativePoints
+
+            points = negativePoints if len(negativePoints) > 0 else positivePoints
+            box = np.array(
+                [
+                    min(x["x"] for x in points),
+                    min(x["y"] for x in points),
+                    max(x["x"] for x in points),
+                    max(x["y"] for x in points),
+                ]
+            )
+            boxesMulti[key] = box
+
+            if pose_landmarks is not None:
+                positiveBreathX = (
+                    (pose_landmarks[11].x + pose_landmarks[12].x) / 2
+                ) * W
+                positiveBreathY = (
+                    (pose_landmarks[11].y + pose_landmarks[12].y) / 2
+                ) * H
+                negativeBreathX1 = pose_landmarks[0].x * W
+                negativeBreathY1 = pose_landmarks[0].y * H
+                negativeBreathX2 = pose_landmarks[9].x * W
+                negativeBreathY2 = pose_landmarks[9].y * H
+                negativeBreathX3 = pose_landmarks[10].x * W
+                negativeBreathY3 = pose_landmarks[10].y * H
+                imagePromptsMulti["breath"] = [
+                    {"x": positiveBreathX, "y": positiveBreathY, "label": 1},
+                    {"x": negativeBreathX1, "y": negativeBreathY1, "label": 0},
+                    {"x": negativeBreathX2, "y": negativeBreathY2, "label": 0},
+                    {"x": negativeBreathX3, "y": negativeBreathY3, "label": 0},
+                ]
+
+        return imagePromptsMulti, boxesMulti
+
+    def detect_face(self, np_image):
+        global face_landmarker, pose_landmarker
+        mp_image = mp.Image(
+            image_format=mp.ImageFormat.SRGB, data=(np_image * 255).astype(np.uint8)
+        )
+        face_landmarks = face_landmarker.detect(mp_image).face_landmarks
+        face_landmarks = face_landmarks[0] if len(face_landmarks) > 0 else None
+        pose_landmarks = pose_landmarker.detect(mp_image).pose_landmarks
+        pose_landmarks = pose_landmarks[0] if len(pose_landmarks) > 0 else None
+        imagePromptsMulti, boxesMulti = self.auto_segment(
+            np_image, face_landmarks, pose_landmarks
+        )
+
+        return imagePromptsMulti, boxesMulti
+
+    def load_image(self, image, ckpt, embedding_id, image_prompts_json):
+        image_prompts = json.loads(image_prompts_json.replace("'", '"'))
+
+        order_file = f"{self.output_dir}/segments_{embedding_id}/order.json"
+        if os.path.exists(order_file):
+            # Frontend uploads segments images to backend => backend reads all segments images and passes them to next nodes
+            with open(order_file) as f:
+                order = json.load(f)
+
+            result = [image_prompts]
+
+            for segment in order:
+                image = Image.open(
+                    f"{self.output_dir}/segments_{embedding_id}/{segment}.png"
+                )
+                image = np.array(image).astype(np.float32) / 255.0
+                image = torch.from_numpy(image)[None,]
+                result.append(image)
+
+            return result
+        else:
+            # Frontend uploads clicks coordinates to backend => backend runs SAM and passes the segments to next nodes
+            model_type = re.findall(r"vit_[lbh]", ckpt)[0]
+
+            global global_predictor
+            if global_predictor is None:
+                global_predictor, _, _ = self.load_models(ckpt, model_type)
+
+            if image.shape[3] == 4:
+                image = image[:, :, :, :3]
+
+            emb_filename = f"{self.output_dir}/{embedding_id}_{model_type}.npy"
+            if not os.path.exists(emb_filename):
+                image_np = (image[0].numpy() * 255).astype(np.uint8)
+                global_predictor.set_image(image_np)
+                emb = global_predictor.get_image_embedding().cpu().numpy()
+                np.save(emb_filename, emb)
+
+                with open(
+                    f"{self.output_dir}/{embedding_id}_{model_type}.json", "w"
+                ) as f:
+                    data = {
+                        "input_size": global_predictor.input_size,
+                        "original_size": global_predictor.original_size,
+                    }
+                    json.dump(data, f)
+            else:
+                emb = np.load(emb_filename)
+
+                with open(f"{self.output_dir}/{embedding_id}_{model_type}.json") as f:
+                    data = json.load(f)
+                    global_predictor.input_size = data["input_size"]
+                    global_predictor.features = torch.from_numpy(emb)
+                    global_predictor.is_image_set = True
+                    global_predictor.original_size = data["original_size"]
+
+            imagePromptsMulti, boxesMulti = self.detect_face(image[0].numpy())
+
+            image_prompts = json.loads(image_prompts_json.replace("'", '"'))
+            result = [image_prompts]
+
+            if isinstance(image_prompts, list):
+                pass
+            elif all(isinstance(item, list) for item in image_prompts.values()):
+                for key, item in image_prompts.items():
+                    if len(item) == 0:
+                        h, w, c = image[0].shape
+                        result.append(torch.zeros(1, h, w, c))
+                        continue
+
+                    points = (
+                        imagePromptsMulti[key] if key in imagePromptsMulti else item
+                    )
+                    point_coords = np.array([[p["x"], p["y"]] for p in points])
+                    point_labels = np.array([p["label"] for p in points])
+
+                    masks, _, _ = global_predictor.predict(
+                        point_coords=point_coords,
+                        point_labels=point_labels,
+                        box=boxesMulti[key] if key in boxesMulti else None,
+                    )
+                    masks = torch.from_numpy(masks)
+                    masks = rearrange(masks[0], "h w -> 1 h w")
+                    out_image = repeat(masks, "1 h w -> 1 h w c", c=3) * image
+                    result.append(out_image)
+            return result
+
+
+NODE_CLASS_MAPPINGS = {"SAM MultiLayer": SAMMultiLayer}
+
+NODE_DISPLAY_NAME_MAPPINGS = {"SAM MultiLayer": "SAM MultiLayer"}

tailwind.config.js

@@ -3,6 +3,9 @@ module.exports = {
   // content: ['./js/**/*.{html,js}'],
   content: ['./js/**/*.{html,js}'],
   theme: {
+    fontFamily: {
+      'gabarito': ['Gabarito'],
+    },
     extend: {},
   },
   daisyui: {

workflow_templates/MouthOpen_(inpaint).json

@@ -1,467 +0,0 @@
-{
-  "last_node_id": 316,
-  "last_link_id": 561,
-  "nodes": [
-    {
-      "id": 181,
-      "type": "CheckpointLoaderSimple",
-      "pos": [
-        40,
-        200
-      ],
-      "size": {
-        "0": 340.6639709472656,
-        "1": 98
-      },
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "MODEL",
-          "type": "MODEL",
-          "links": [
-            552
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "CLIP",
-          "type": "CLIP",
-          "links": [
-            527,
-            558
-          ],
-          "shape": 3,
-          "slot_index": 1
-        },
-        {
-          "name": "VAE",
-          "type": "VAE",
-          "links": [
-            249,
-            547
-          ],
-          "shape": 3,
-          "slot_index": 2
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CheckpointLoaderSimple"
-      },
-      "widgets_values": [
-        "meinamix_meinaV10.safetensors"
-      ]
-    },
-    {
-      "id": 176,
-      "type": "KSampler",
-      "pos": [
-        900,
-        230
-      ],
-      "size": {
-        "0": 326.9862060546875,
-        "1": 262.3746643066406
-      },
-      "flags": {},
-      "order": 5,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "model",
-          "type": "MODEL",
-          "link": 552
-        },
-        {
-          "name": "positive",
-          "type": "CONDITIONING",
-          "link": 557,
-          "slot_index": 1
-        },
-        {
-          "name": "negative",
-          "type": "CONDITIONING",
-          "link": 554
-        },
-        {
-          "name": "latent_image",
-          "type": "LATENT",
-          "link": 540,
-          "slot_index": 3
-        }
-      ],
-      "outputs": [
-        {
-          "name": "LATENT",
-          "type": "LATENT",
-          "links": [
-            248
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "KSampler"
-      },
-      "widgets_values": [
-        509878269555324,
-        "randomize",
-        25,
-        7,
-        "dpmpp_2m",
-        "karras",
-        1
-      ]
-    },
-    {
-      "id": 175,
-      "type": "CLIPTextEncode",
-      "pos": [
-        540,
-        300
-      ],
-      "size": {
-        "0": 304.70074462890625,
-        "1": 109.89002990722656
-      },
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "clip",
-          "type": "CLIP",
-          "link": 527
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            554
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CLIPTextEncode"
-      },
-      "widgets_values": [
-        "worst quality,low quality, normal quality, monochrome, grayscale, closed mouth, extended tongue,sad"
-      ],
-      "color": "#322",
-      "bgcolor": "#533"
-    },
-    {
-      "id": 174,
-      "type": "CLIPTextEncode",
-      "pos": [
-        540,
-        130
-      ],
-      "size": {
-        "0": 299.78692626953125,
-        "1": 101.67720031738281
-      },
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "clip",
-          "type": "CLIP",
-          "link": 558,
-          "slot_index": 0
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            557
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CLIPTextEncode"
-      },
-      "widgets_values": [
-        "open mouth,detailed mouth,lips,smile"
-      ],
-      "color": "#232",
-      "bgcolor": "#353"
-    },
-    {
-      "id": 301,
-      "type": "VAEEncodeForInpaint",
-      "pos": [
-        538,
-        470
-      ],
-      "size": {
-        "0": 315,
-        "1": 98
-      },
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "pixels",
-          "type": "IMAGE",
-          "link": 559,
-          "slot_index": 0
-        },
-        {
-          "name": "vae",
-          "type": "VAE",
-          "link": 547,
-          "slot_index": 1
-        },
-        {
-          "name": "mask",
-          "type": "MASK",
-          "link": 560,
-          "slot_index": 2
-        }
-      ],
-      "outputs": [
-        {
-          "name": "LATENT",
-          "type": "LATENT",
-          "links": [
-            540
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VAEEncodeForInpaint"
-      },
-      "widgets_values": [
-        6
-      ]
-    },
-    {
-      "id": 315,
-      "type": "LoadImage",
-      "pos": [
-        61,
-        455
-      ],
-      "size": {
-        "0": 315,
-        "1": 314.0001525878906
-      },
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            559
-          ],
-          "shape": 3
-        },
-        {
-          "name": "MASK",
-          "type": "MASK",
-          "links": [
-            560
-          ],
-          "shape": 3,
-          "slot_index": 1
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadImage"
-      },
-      "widgets_values": [
-        "clipspace/clipspace-mask-186477.83499991894.png [input]",
-        "image"
-      ]
-    },
-    {
-      "id": 184,
-      "type": "VAEDecode",
-      "pos": [
-        1300,
-        230
-      ],
-      "size": {
-        "0": 210,
-        "1": 46
-      },
-      "flags": {
-        "collapsed": false
-      },
-      "order": 6,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "samples",
-          "type": "LATENT",
-          "link": 248
-        },
-        {
-          "name": "vae",
-          "type": "VAE",
-          "link": 249,
-          "slot_index": 1
-        }
-      ],
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            561
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VAEDecode"
-      }
-    },
-    {
-      "id": 316,
-      "type": "PreviewImage",
-      "pos": [
-        1571.9339136962892,
-        243.9844624023438
-      ],
-      "size": {
-        "0": 210,
-        "1": 26
-      },
-      "flags": {},
-      "order": 7,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "images",
-          "type": "IMAGE",
-          "link": 561
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "PreviewImage"
-      }
-    }
-  ],
-  "links": [
-    [
-      248,
-      176,
-      0,
-      184,
-      0,
-      "LATENT"
-    ],
-    [
-      249,
-      181,
-      2,
-      184,
-      1,
-      "VAE"
-    ],
-    [
-      527,
-      181,
-      1,
-      175,
-      0,
-      "CLIP"
-    ],
-    [
-      540,
-      301,
-      0,
-      176,
-      3,
-      "LATENT"
-    ],
-    [
-      547,
-      181,
-      2,
-      301,
-      1,
-      "VAE"
-    ],
-    [
-      552,
-      181,
-      0,
-      176,
-      0,
-      "MODEL"
-    ],
-    [
-      554,
-      175,
-      0,
-      176,
-      2,
-      "CONDITIONING"
-    ],
-    [
-      557,
-      174,
-      0,
-      176,
-      1,
-      "CONDITIONING"
-    ],
-    [
-      558,
-      181,
-      1,
-      174,
-      0,
-      "CLIP"
-    ],
-    [
-      559,
-      315,
-      0,
-      301,
-      0,
-      "IMAGE"
-    ],
-    [
-      560,
-      315,
-      1,
-      301,
-      2,
-      "MASK"
-    ],
-    [
-      561,
-      184,
-      0,
-      316,
-      0,
-      "IMAGE"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {},
-  "version": 0.4
-}

workflow_templates/SimpleCharacterGen.json

@@ -1,383 +0,0 @@
-{
-  "last_node_id": 318,
-  "last_link_id": 563,
-  "nodes": [
-    {
-      "id": 184,
-      "type": "VAEDecode",
-      "pos": [
-        1290,
-        410
-      ],
-      "size": {
-        "0": 210,
-        "1": 46
-      },
-      "flags": {
-        "collapsed": false
-      },
-      "order": 5,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "samples",
-          "type": "LATENT",
-          "link": 248
-        },
-        {
-          "name": "vae",
-          "type": "VAE",
-          "link": 249,
-          "slot_index": 1
-        }
-      ],
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            562
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VAEDecode"
-      }
-    },
-    {
-      "id": 317,
-      "type": "SaveImage",
-      "pos": [
-        1530,
-        420
-      ],
-      "size": [
-        315,
-        270
-      ],
-      "flags": {},
-      "order": 6,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "images",
-          "type": "IMAGE",
-          "link": 562
-        }
-      ],
-      "properties": {},
-      "widgets_values": [
-        "ComfyUI"
-      ]
-    },
-    {
-      "id": 175,
-      "type": "CLIPTextEncode",
-      "pos": [
-        530,
-        480
-      ],
-      "size": [
-        301.38145862926115,
-        151.58555811101718
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "clip",
-          "type": "CLIP",
-          "link": 527
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            554
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CLIPTextEncode"
-      },
-      "widgets_values": [
-        "worst quality, low quality, normal quality, monochrome, grayscale, closed mouth, extended tongue, sad, text, signature, username, artist name, animal, big mouth,leg,hand"
-      ],
-      "color": "#322",
-      "bgcolor": "#533"
-    },
-    {
-      "id": 181,
-      "type": "CheckpointLoaderSimple",
-      "pos": [
-        100,
-        390
-      ],
-      "size": {
-        "0": 340.6639709472656,
-        "1": 98
-      },
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "MODEL",
-          "type": "MODEL",
-          "links": [
-            552
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "CLIP",
-          "type": "CLIP",
-          "links": [
-            527,
-            558
-          ],
-          "shape": 3,
-          "slot_index": 1
-        },
-        {
-          "name": "VAE",
-          "type": "VAE",
-          "links": [
-            249
-          ],
-          "shape": 3,
-          "slot_index": 2
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CheckpointLoaderSimple"
-      },
-      "widgets_values": [
-        "meinamix_meinaV11.safetensors"
-      ]
-    },
-    {
-      "id": 174,
-      "type": "CLIPTextEncode",
-      "pos": [
-        530,
-        300
-      ],
-      "size": [
-        294.7698808149855,
-        129.02360742742354
-      ],
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "clip",
-          "type": "CLIP",
-          "link": 558,
-          "slot_index": 0
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            557
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CLIPTextEncode"
-      },
-      "widgets_values": [
-        "looking at viewer, detailed face, open mouth, smile, solo,eye-level angle,\nsimple background, 1 girl, half body"
-      ],
-      "color": "#232",
-      "bgcolor": "#353"
-    },
-    {
-      "id": 318,
-      "type": "EmptyLatentImage",
-      "pos": [
-        530,
-        690
-      ],
-      "size": [
-        299.24921209161903,
-        106
-      ],
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "LATENT",
-          "type": "LATENT",
-          "links": [
-            563
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "EmptyLatentImage"
-      },
-      "widgets_values": [
-        512,
-        768,
-        1
-      ]
-    },
-    {
-      "id": 176,
-      "type": "KSampler",
-      "pos": [
-        890,
-        410
-      ],
-      "size": {
-        "0": 326.9862060546875,
-        "1": 262.3746643066406
-      },
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "model",
-          "type": "MODEL",
-          "link": 552
-        },
-        {
-          "name": "positive",
-          "type": "CONDITIONING",
-          "link": 557,
-          "slot_index": 1
-        },
-        {
-          "name": "negative",
-          "type": "CONDITIONING",
-          "link": 554
-        },
-        {
-          "name": "latent_image",
-          "type": "LATENT",
-          "link": 563,
-          "slot_index": 3
-        }
-      ],
-      "outputs": [
-        {
-          "name": "LATENT",
-          "type": "LATENT",
-          "links": [
-            248
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "KSampler"
-      },
-      "widgets_values": [
-        24285104147823,
-        "randomize",
-        25,
-        5,
-        "dpmpp_2m",
-        "karras",
-        1
-      ]
-    }
-  ],
-  "links": [
-    [
-      248,
-      176,
-      0,
-      184,
-      0,
-      "LATENT"
-    ],
-    [
-      249,
-      181,
-      2,
-      184,
-      1,
-      "VAE"
-    ],
-    [
-      527,
-      181,
-      1,
-      175,
-      0,
-      "CLIP"
-    ],
-    [
-      552,
-      181,
-      0,
-      176,
-      0,
-      "MODEL"
-    ],
-    [
-      554,
-      175,
-      0,
-      176,
-      2,
-      "CONDITIONING"
-    ],
-    [
-      557,
-      174,
-      0,
-      176,
-      1,
-      "CONDITIONING"
-    ],
-    [
-      558,
-      181,
-      1,
-      174,
-      0,
-      "CLIP"
-    ],
-    [
-      562,
-      184,
-      0,
-      317,
-      0,
-      "IMAGE"
-    ],
-    [
-      563,
-      318,
-      0,
-      176,
-      3,
-      "LATENT"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {},
-  "version": 0.4
-}

workflow_templates/TemplateGen01/TemplateGen01.json

@@ -1,669 +0,0 @@
-{
-  "last_node_id": 23,
-  "last_link_id": 27,
-  "nodes": [
-    {
-      "id": 19,
-      "type": "ControlNetApply",
-      "pos": [
-        660,
-        300
-      ],
-      "size": {
-        "0": 317.4000244140625,
-        "1": 98
-      },
-      "flags": {},
-      "order": 8,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "conditioning",
-          "type": "CONDITIONING",
-          "link": 23
-        },
-        {
-          "name": "control_net",
-          "type": "CONTROL_NET",
-          "link": 22,
-          "slot_index": 1
-        },
-        {
-          "name": "image",
-          "type": "IMAGE",
-          "link": 26,
-          "slot_index": 2
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            24
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "ControlNetApply"
-      },
-      "widgets_values": [
-        0.75
-      ]
-    },
-    {
-      "id": 7,
-      "type": "ControlNetApply",
-      "pos": [
-        1020,
-        300
-      ],
-      "size": {
-        "0": 317.4000244140625,
-        "1": 98
-      },
-      "flags": {},
-      "order": 9,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "conditioning",
-          "type": "CONDITIONING",
-          "link": 24,
-          "slot_index": 0
-        },
-        {
-          "name": "control_net",
-          "type": "CONTROL_NET",
-          "link": 8,
-          "slot_index": 1
-        },
-        {
-          "name": "image",
-          "type": "IMAGE",
-          "link": 7
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            9
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "ControlNetApply"
-      },
-      "widgets_values": [
-        0.75
-      ]
-    },
-    {
-      "id": 13,
-      "type": "VAEDecode",
-      "pos": [
-        1740,
-        309
-      ],
-      "size": [
-        320.81564830322213,
-        46
-      ],
-      "flags": {},
-      "order": 11,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "samples",
-          "type": "LATENT",
-          "link": 15
-        },
-        {
-          "name": "vae",
-          "type": "VAE",
-          "link": 16,
-          "slot_index": 1
-        }
-      ],
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            27
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "VAEDecode"
-      }
-    },
-    {
-      "id": 23,
-      "type": "SaveImage",
-      "pos": [
-        1738,
-        413
-      ],
-      "size": [
-        317.3241088867178,
-        288.2958862304681
-      ],
-      "flags": {},
-      "order": 12,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "images",
-          "type": "IMAGE",
-          "link": 27
-        }
-      ],
-      "properties": {},
-      "widgets_values": [
-        "Avatar"
-      ]
-    },
-    {
-      "id": 10,
-      "type": "CLIPTextEncode",
-      "pos": [
-        219,
-        301
-      ],
-      "size": [
-        398.1301946411129,
-        115.324566581726
-      ],
-      "flags": {},
-      "order": 6,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "clip",
-          "type": "CLIP",
-          "link": 11,
-          "slot_index": 0
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            23
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CLIPTextEncode"
-      },
-      "widgets_values": [
-        "looking at viewer, detailed face, open mouth, [smile], solo,eye-level angle, upper body, half body"
-      ],
-      "color": "#232",
-      "bgcolor": "#353"
-    },
-    {
-      "id": 12,
-      "type": "CLIPTextEncode",
-      "pos": [
-        221,
-        469
-      ],
-      "size": [
-        395.5002463378903,
-        111.47454442596415
-      ],
-      "flags": {},
-      "order": 7,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "clip",
-          "type": "CLIP",
-          "link": 14,
-          "slot_index": 0
-        }
-      ],
-      "outputs": [
-        {
-          "name": "CONDITIONING",
-          "type": "CONDITIONING",
-          "links": [
-            13
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CLIPTextEncode"
-      },
-      "widgets_values": [
-        "nsfw, worst quality, low quality, normal quality, monochrome, grayscale, closed mouth, extended tongue, sad, text, signature, username, artist name, animal, big mouth, leg, hand, missing body, ugly, blurry"
-      ],
-      "color": "#322",
-      "bgcolor": "#533"
-    },
-    {
-      "id": 11,
-      "type": "CheckpointLoaderSimple",
-      "pos": [
-        -150,
-        307
-      ],
-      "size": [
-        324.9937083129885,
-        98
-      ],
-      "flags": {},
-      "order": 4,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "MODEL",
-          "type": "MODEL",
-          "links": [
-            12
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "CLIP",
-          "type": "CLIP",
-          "links": [
-            11,
-            14
-          ],
-          "shape": 3
-        },
-        {
-          "name": "VAE",
-          "type": "VAE",
-          "links": [
-            16
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "CheckpointLoaderSimple"
-      },
-      "widgets_values": [
-        "meinamix_meinaV10.safetensors"
-      ]
-    },
-    {
-      "id": 22,
-      "type": "LoadImage",
-      "pos": [
-        653,
-        461
-      ],
-      "size": [
-        315,
-        314
-      ],
-      "flags": {},
-      "order": 2,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            26
-          ],
-          "shape": 3
-        },
-        {
-          "name": "MASK",
-          "type": "MASK",
-          "links": null,
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadImage"
-      },
-      "widgets_values": [
-        "ComfyUI_temp_nycbp_00001_.png",
-        "image"
-      ]
-    },
-    {
-      "id": 1,
-      "type": "LoadImage",
-      "pos": [
-        1020,
-        468
-      ],
-      "size": [
-        315,
-        314.00001525878906
-      ],
-      "flags": {},
-      "order": 0,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "IMAGE",
-          "type": "IMAGE",
-          "links": [
-            7
-          ],
-          "shape": 3,
-          "slot_index": 0
-        },
-        {
-          "name": "MASK",
-          "type": "MASK",
-          "links": null,
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "LoadImage"
-      },
-      "widgets_values": [
-        "ComfyUI_temp_rlaph_00001_s.png",
-        "image"
-      ]
-    },
-    {
-      "id": 9,
-      "type": "KSampler",
-      "pos": [
-        1379,
-        305
-      ],
-      "size": {
-        "0": 315,
-        "1": 262
-      },
-      "flags": {},
-      "order": 10,
-      "mode": 0,
-      "inputs": [
-        {
-          "name": "model",
-          "type": "MODEL",
-          "link": 12
-        },
-        {
-          "name": "positive",
-          "type": "CONDITIONING",
-          "link": 9
-        },
-        {
-          "name": "negative",
-          "type": "CONDITIONING",
-          "link": 13,
-          "slot_index": 2
-        },
-        {
-          "name": "latent_image",
-          "type": "LATENT",
-          "link": 18,
-          "slot_index": 3
-        }
-      ],
-      "outputs": [
-        {
-          "name": "LATENT",
-          "type": "LATENT",
-          "links": [
-            15
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "KSampler"
-      },
-      "widgets_values": [
-        291874885197794,
-        "randomize",
-        25,
-        7,
-        "dpmpp_2m",
-        "karras",
-        1
-      ]
-    },
-    {
-      "id": 8,
-      "type": "ControlNetLoader",
-      "pos": [
-        1020,
-        190
-      ],
-      "size": {
-        "0": 315,
-        "1": 58
-      },
-      "flags": {},
-      "order": 3,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "CONTROL_NET",
-          "type": "CONTROL_NET",
-          "links": [
-            8
-          ],
-          "shape": 3,
-          "slot_index": 0
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "ControlNetLoader"
-      },
-      "widgets_values": [
-        "control_v11p_sd15_normalbae_fp16.safetensors"
-      ]
-    },
-    {
-      "id": 20,
-      "type": "ControlNetLoader",
-      "pos": [
-        665,
-        190
-      ],
-      "size": {
-        "0": 315,
-        "1": 58
-      },
-      "flags": {},
-      "order": 1,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "CONTROL_NET",
-          "type": "CONTROL_NET",
-          "links": [
-            22
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "ControlNetLoader"
-      },
-      "widgets_values": [
-        "control_v11p_sd15_openpose_fp16.safetensors"
-      ]
-    },
-    {
-      "id": 15,
-      "type": "EmptyLatentImage",
-      "pos": [
-        1377,
-        150
-      ],
-      "size": {
-        "0": 315,
-        "1": 106
-      },
-      "flags": {},
-      "order": 5,
-      "mode": 0,
-      "outputs": [
-        {
-          "name": "LATENT",
-          "type": "LATENT",
-          "links": [
-            18
-          ],
-          "shape": 3
-        }
-      ],
-      "properties": {
-        "Node name for S&R": "EmptyLatentImage"
-      },
-      "widgets_values": [
-        512,
-        768,
-        1
-      ]
-    }
-  ],
-  "links": [
-    [
-      7,
-      1,
-      0,
-      7,
-      2,
-      "IMAGE"
-    ],
-    [
-      8,
-      8,
-      0,
-      7,
-      1,
-      "CONTROL_NET"
-    ],
-    [
-      9,
-      7,
-      0,
-      9,
-      1,
-      "CONDITIONING"
-    ],
-    [
-      11,
-      11,
-      1,
-      10,
-      0,
-      "CLIP"
-    ],
-    [
-      12,
-      11,
-      0,
-      9,
-      0,
-      "MODEL"
-    ],
-    [
-      13,
-      12,
-      0,
-      9,
-      2,
-      "CONDITIONING"
-    ],
-    [
-      14,
-      11,
-      1,
-      12,
-      0,
-      "CLIP"
-    ],
-    [
-      15,
-      9,
-      0,
-      13,
-      0,
-      "LATENT"
-    ],
-    [
-      16,
-      11,
-      2,
-      13,
-      1,
-      "VAE"
-    ],
-    [
-      18,
-      15,
-      0,
-      9,
-      3,
-      "LATENT"
-    ],
-    [
-      22,
-      20,
-      0,
-      19,
-      1,
-      "CONTROL_NET"
-    ],
-    [
-      23,
-      10,
-      0,
-      19,
-      0,
-      "CONDITIONING"
-    ],
-    [
-      24,
-      19,
-      0,
-      7,
-      0,
-      "CONDITIONING"
-    ],
-    [
-      26,
-      22,
-      0,
-      19,
-      2,
-      "IMAGE"
-    ],
-    [
-      27,
-      13,
-      0,
-      23,
-      0,
-      "IMAGE"
-    ]
-  ],
-  "groups": [],
-  "config": {},
-  "extra": {},
-  "version": 0.4
-}