add her demo

README.md

@@ -8,6 +8,8 @@ For business cooperation, please contact email 389570357@qq.com
 
 ##### `最新`：
 
+- [增加 Her 的DEMO页面，和数字人对话](https://github.com/shadowcz007/ComfyUI-Backend-MixlabNodes/blob/main/workflow/her_demo_workflow.json)
+
 - 右键菜单支持 text-to-text，方便对 prompt 词补全，支持云LLM或者是本地LLM。
 
 - 增加 MiniCPM-V 2.6 int4

__init__.py

@@ -867,7 +867,7 @@ def random_seed(seed, data):
             if value.get('class_type') == "Seed_" and seed[id] in ['increment', 'decrement', 'randomize']:
                 value['inputs']['seed'] = round(random.random() * max_seed)
             
-        print('new Seed', value)
+        # print('new Seed', value)
     
     return data
 
@@ -983,7 +983,9 @@ async def handle_ar_page(request):
 # 重启服务
 @routes.post('/mixlab/re_start')
 def re_start(request):
+    p_intance=PromptServer.instance
     try:
+        p_intance.prompt_queue.set_flag("free_memory", True)
         sys.stdout.close_log()
     except Exception as e:
         pass

nodes/ImageNode.py

@@ -19,7 +19,6 @@ from .Watcher import FolderWatcher
 
 from itertools import product
 
-
 # 将PIL图片转换为OpenCV格式
 def pil_to_opencv(image):
     open_cv_image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
@@ -44,14 +43,13 @@ def get_files_with_extension(directory, extensions):
                 # 直接将文件名添加到列表中
                 file_list.append(file)
     return file_list
-	
 def composite_images(foreground, background, mask, is_multiply_blend=False, position="overall", scale=0.25):
     width, height = foreground.size
     bg_image = background
     bwidth, bheight = bg_image.size
 
-    scale=max(scale,1/bwidth)
-    scale=max(scale,1/bheight)
+    scale = max(scale, 1 / bwidth)
+    scale = max(scale, 1 / bheight)
 
     def determine_scale_option(width, height):
         return 'height' if height > width else 'width'
@@ -70,9 +68,9 @@ def composite_images(foreground, background, mask, is_multiply_blend=False, posi
     else:
         scale_option = determine_scale_option(width, height)
         if scale_option == 'height':
-            scale = int(bheight * scale) / height
+            scale = bheight * scale / height
         else:
-            scale = int(bwidth * scale) / width
+            scale = bwidth * scale / width
 
         new_width = int(width * scale)
         new_height = int(height * scale)
@@ -110,22 +108,13 @@ def composite_images(foreground, background, mask, is_multiply_blend=False, posi
             "mask": mask
         }
 
-    layer_image = layer['image']
-    layer_mask = layer['mask']
+    # Resize the foreground image with antialiasing
+    layer_image = layer['image'].resize((layer['width'], layer['height']), Image.ANTIALIAS)
+    layer_mask = layer['mask'].resize((layer['width'], layer['height']), Image.ANTIALIAS)
 
-    bg_image = merge_images(bg_image,
-                            layer_image,
-                            layer_mask,
-                            layer['x'],
-                            layer['y'],
-                            layer['width'],
-                            layer['height'],
-                            layer['scale_option'],
-                            is_multiply_blend)
+    bg_image.paste(layer_image, (layer['x'], layer['y']), layer_mask)
 
-    bg_image = bg_image.convert('RGB')
-
-    return bg_image
+    return bg_image.convert('RGB')
 
 
 
@@ -1390,8 +1379,8 @@ class LoadImagesFromPath:
                     }),
                     "watcher":(["disable","enable"],),
                     "result": ("WATCHER",),#为了激活本节点运行
-                     "prompt": ("PROMPT",),
-                    # "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
+                    "prompt": ("PROMPT",),
+                    "seed": (any_type,  {"default": 0, "min": 0, "max": 0xffffffffffffffff}),
                 }
             }
     
@@ -1409,7 +1398,7 @@ class LoadImagesFromPath:
     watcher_folder=None
 
     # 运行的函数
-    def run(self,file_path,white_bg,newest_files,index_variable,watcher,result,prompt):
+    def run(self,file_path,white_bg,newest_files,index_variable,watcher,result,prompt,seed=1):
         global watcher_folder
         # print('###监听:',watcher_folder,watcher,file_path,result)
 
@@ -1457,7 +1446,8 @@ class LoadImagesFromPath:
             print("发生了一个未知的错误：", str(e))
 
         # print('#prompt::::',prompt)
-        return  {"ui": {"seed": [1]}, "result":(imgs,masks,prompt,file_names,)}
+        # return  {"ui": {"seed": [1]}, "result":(imgs,masks,prompt,file_names,)}
+        return  (imgs,masks,prompt,file_names,)
 
 
 # TODO 扩大选区的功能,重新输出mask
@@ -3301,7 +3291,6 @@ class DepthViewer_:
             "required": {
                 "image": ("IMAGE",),
                 "depth_map": ("IMAGE",),
-
             },
              "optional":{
                   "frames":("IMAGEBASE64",), 
@@ -3316,7 +3305,6 @@ class DepthViewer_:
             "imagesave", 
             folder_paths.get_output_directory())
 
-
     RETURN_TYPES = ("IMAGE",)
     RETURN_NAMES = ("frames",)
     
@@ -3366,14 +3354,20 @@ class DepthViewer_:
         image1 = Image.new('RGB', (512, 512), color='black')
         image1=pil2tensor(image1)
 
-        if frames!=None:
+        # print('frames',frames)
+        if frames!=None and "images" in frames:
+            
             for im in frames['images']:
                 # print(im)
                 if 'type' in im and (not f"[{im['type']}]" in im['name']):
                     im['name']=im['name']+" "+f"[{im['type']}]"
-                    
-                output_image, output_mask = load_image_to_tensor(im['name'])
-                ims.append(output_image)
+                
+                try:
+                    output_image, output_mask = load_image_to_tensor(im['name'])
+                    ims.append(output_image)
+                except:
+                    print("no")
+                
 
             if len(ims)>0:
                 image1 = ims[0]

nodes/Utils.py

@@ -82,13 +82,13 @@ def tensor2pil(image):
     return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8))
 
 
-def create_temp_file(image):
+def create_temp_file(image,counter=1):
     output_dir = folder_paths.get_temp_directory()
 
     (
             full_output_folder,
             filename,
-            counter,
+            _,
             subfolder,
             _,
         ) = folder_paths.get_save_image_path('tmp', output_dir)
@@ -601,6 +601,7 @@ class AppInfo:
                     "link":("STRING",{"multiline": False,"default": "https://","dynamicPrompts": False}),
                     "category":("STRING",{"multiline": False,"default": "","dynamicPrompts": False}),
                     "auto_save": (["enable","disable"],),
+                    "idle_animation": ("BOOLEAN", {"default": False},),
                 }
 
                 }
@@ -616,14 +617,19 @@ class AppInfo:
     INPUT_IS_LIST = True
     # OUTPUT_IS_LIST = (True,)
 
-    def run(self,name,input_ids,output_ids,image,description,version,share_prefix,link,category,auto_save):
+    def run(self,name,input_ids,output_ids,image,description,version,share_prefix,link,category,auto_save,idle_animation):
         name=name[0]
+
+        idle_animation=idle_animation[0]
         
-        im=None
+        im=[]
         if image:
-            im=image[0][0]
-            #TODO batch 的方式需要处理
-            im=create_temp_file(im)
+            # img=image[0][0]
+            print('AppInfo_image',len(image))
+            # batch 的方式需要处理
+            for i in range(len(image)):
+                img=image[i]
+                im.append(create_temp_file(img,i+1)[0])
         # image [img,] img[batch,w,h,a] 列表里面是batch，
 
         input_ids=input_ids[0]
@@ -636,7 +642,7 @@ class AppInfo:
         
         # id=get_json_hash([name,im,input_ids,output_ids,description,version])
 
-        return {"ui": {"json": [name,im,input_ids,output_ids,description,version,share_prefix,link,category]}, "result": ()}
+        return {"ui": {"json": [name,im,input_ids,output_ids,description,version,share_prefix,link,category,idle_animation]}, "result": ()}
     
 
     

nodes/Video.py

@@ -913,7 +913,7 @@ class GenerateFramesByCount:
     def r(self, frames, frame_count, revert):
         
         image_list = [frames[i:i + 1, ...] for i in range(frames.shape[0])]
-
+        print('#image_list',len(image_list),frame_count)
         image_list=get_frames(frame_count,image_list,revert)
 
         images = torch.cat(image_list, dim=0)

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-mixlab-nodes"
 description = "3D, ScreenShareNode & FloatingVideoNode, SpeechRecognition & SpeechSynthesis, GPT, LoadImagesFromLocal, Layers, Other Nodes, ..."
-version = "0.40.0"
+version = "0.41.0"
 license = "MIT"
 dependencies = ["numpy", "pyOpenSSL", "watchdog", "opencv-python-headless", "matplotlib", "openai", "simple-lama-inpainting", "clip-interrogator==0.6.0", "transformers>=4.36.0", "lark-parser", "imageio-ffmpeg", "rembg[gpu]", "omegaconf==2.3.0", "Pillow>=9.5.0", "einops==0.7.0", "trimesh>=4.0.5", "huggingface-hub", "scikit-image"]
 

requirements.txt

@@ -4,19 +4,19 @@ watchdog
 opencv-python-headless
 matplotlib
 openai
+torchaudio
 # simple-lama-inpainting
 clip-interrogator==0.6.0
 transformers>=4.36.0
 lark-parser
 imageio-ffmpeg
 rembg[gpu]
-omegaconf==2.3.0
+omegaconf>=2.3.0
 Pillow>=9.5.0
-einops==0.7.0
+einops>=0.7.0
 trimesh>=4.0.5
 huggingface-hub
 scikit-image
-torchaudio
 soundfile>=0.12.1
 json-repair
 

web/javascript/3d_mixlab.js

@@ -2,7 +2,61 @@ import { app } from '../../../scripts/app.js'
 import { api } from '../../../scripts/api.js'
 import { $el } from '../../../scripts/ui.js'
 
-import { loadExternalScript } from './common.js'
+import { loadExternalScript, get_position_style } from './common.js'
+
+function setCameraOrbit (modelview, distant, angles, screenNumber) {
+  //2.1 20
+  // const angles = {
+  //   1: -20.0,
+  //   2: -17.9,
+  //   3: -15.8,
+  //   4: -13.7,
+  //   5: -11.6,
+  //   6: -9.5,
+  //   7: -7.4,
+  //   8: -5.3,
+  //   9: -3.2,
+  //   10: -1.1,
+  //   11: 1.1,
+  //   12: 3.2,
+  //   13: 5.3,
+  //   14: 7.4,
+  //   15: 9.5,
+  //   16: 11.6,
+  //   17: 13.7,
+  //   18: 15.8,
+  //   19: 17.9,
+  //   20: 20.0
+  // };
+
+  // 12 3.6
+  // const angles = {
+  //   1: -20.0,
+  //   2: -16.4,
+  //   3: -12.7,
+  //   4: -9.1,
+  //   5: -5.5,
+  //   6: -1.8,
+  //   7: 1.8,
+  //   8: 5.5,
+  //   9: 9.1,
+  //   10: 12.7,
+  //   11: 16.4,
+  //   12: 20.0
+  // }
+
+  const angle = angles[screenNumber]
+
+  let co=modelview.cameraOrbit.split(" ")
+
+  if (angle !== undefined) {
+    
+    modelview.cameraOrbit = `${angle}deg ${co[1]} ${distant}m`
+    console.log(screenNumber, angle)
+  } else {
+    console.error('Invalid screen number')
+  }
+}
 
 const getLocalData = key => {
   let data = {}
@@ -85,41 +139,6 @@ const parseImage = url => {
   })
 }
 
-function get_position_style (ctx, widget_width, y, node_height) {
-  const MARGIN = 4 // the margin around the html element
-
-  /* Create a transform that deals with all the scrolling and zooming */
-  const elRect = ctx.canvas.getBoundingClientRect()
-  const transform = new DOMMatrix()
-    .scaleSelf(
-      elRect.width / ctx.canvas.width,
-      elRect.height / ctx.canvas.height
-    )
-    .multiplySelf(ctx.getTransform())
-    .translateSelf(MARGIN, MARGIN + y)
-
-  return {
-    transformOrigin: '0 0',
-    transform: transform,
-    left:
-      document.querySelector('.comfy-menu').style.display === 'none'
-        ? `60px`
-        : `0`,
-    top: `0`,
-    cursor: 'pointer',
-    position: 'absolute',
-    maxWidth: `${widget_width - MARGIN * 2}px`,
-    // maxHeight: `${node_height - MARGIN * 2}px`, // we're assuming we have the whole height of the node
-    width: `${widget_width - MARGIN * 2}px`,
-    // height: `${node_height * 0.3 - MARGIN * 2}px`,
-    // background: '#EEEEEE',
-    display: 'flex',
-    flexDirection: 'column',
-    // alignItems: 'center',
-    justifyContent: 'space-around'
-  }
-}
-
 async function extractMaterial (
   modelViewerVariants,
   selectMaterial,
@@ -267,33 +286,30 @@ app.registerExtension({
     }
   },
 
+  async init () {
+    await loadExternalScript('/mixlab/app/lib/model-viewer.min.js', 'module')
+  },
+
   async beforeRegisterNodeDef (nodeType, nodeData, app) {
     if (nodeType.comfyClass == '3DImage') {
+      console.log('nodeType.comfyClass', nodeType.comfyClass)
       const orig_nodeCreated = nodeType.prototype.onNodeCreated
       nodeType.prototype.onNodeCreated = async function () {
-        await loadExternalScript(
-          '/mixlab/app/lib/model-viewer.min.js',
-          'module'
-        )
-
         orig_nodeCreated?.apply(this, arguments)
-
         const uploadWidget = this.widgets.filter(w => w.name == 'upload')[0]
-
         const widget = {
           type: 'div',
           name: 'upload-preview',
           draw (ctx, node, widget_width, y, widget_height) {
             Object.assign(
               this.div.style,
-              get_position_style(ctx, widget_width, 88, node.size[1])
+              get_position_style(ctx, widget_width - 122, 88, node.size[1], 44)
             )
           }
         }
 
         widget.div = $el('div', {})
         widget.div.style.width = `120px`
-
         document.body.appendChild(widget.div)
 
         const inputDiv = (key, placeholder, preview) => {
@@ -321,7 +337,11 @@ app.registerExtension({
             // console.log('文件URL: ', fileURL)
             let html = `<model-viewer  src="${fileURL}" 
                 oncontextmenu="return false;"
-                min-field-of-view="0deg" max-field-of-view="180deg"
+                style="outline:1px solid white"
+                min-field-of-view="0deg" 
+                max-field-of-view="180deg"
+                min-camera-orbit="auto auto 0m"
+                max-camera-orbit="auto auto 1000m"
                  shadow-intensity="1" 
                  camera-controls 
                  touch-action="pan-y">
@@ -335,10 +355,9 @@ app.registerExtension({
                    
                   </div>
                   <div>
-                   <input class="ddcap_step" type="number" min="1" max="20" step="1" value="1">
-                   <input class="total_images" type="number" min="1" max="180" step="1" value="40">
-                   <input class="ddcap_range" type="range" min="-180" max="180" step="1" value="0">
-                   <input class="ddcap_range_top" type="range" min="-180" max="180" step="1" value="0">
+                   <input class="ddcap_distant" type="number" min="1" step="1" value="55">
+                   <input class="total_images" type="number" min="1" max="180" step="1" value="20">
+                   <input class="ddcap_range" type="number" min="0" max="20" step="0.1" value="2.1"> 
                   <button class="ddcap">Capture Rotational Screenshots</button></div>
                   
                   <div><button class="export">Export GLB</button></div>
@@ -359,10 +378,9 @@ app.registerExtension({
 
             const exportGLB = preview.querySelector('.export')
 
-            const ddcap_step = preview.querySelector('.ddcap_step')
+            const ddcap_distant = preview.querySelector('.ddcap_distant')
             const total_images = preview.querySelector('.total_images')
             const ddcap_range = preview.querySelector('.ddcap_range')
-            const ddcap_range_top = preview.querySelector('.ddcap_range_top')
             const ddCap = preview.querySelector('.ddcap')
             const sleep = (t = 1000) => {
               return new Promise((res, rej) => {
@@ -383,64 +401,77 @@ app.registerExtension({
               return await uploadImage_(blob, '.png')
             }
 
-            async function captureImages (angleIncrement = 1, totalImages = 12) {
-              // 记录初始旋转角度
-              const initialCameraOrbit =
-                modelViewerVariants.cameraOrbit.split(' ')
-              console.log(
-                '#captureImages',
-                initialCameraOrbit,
-                angleIncrement * totalImages
-              )
-              // const totalImages = 12
-              // const angleIncrement = totalRotation / totalImages // Each increment in degrees
-              let currentAngle =
-                Number(initialCameraOrbit[0].replace('deg', '')) -
-                (angleIncrement * totalImages) / 2 // Start from the leftmost angle
+            async function captureImages (
+              ddcap_range = 1,
+              total_images = 12,
+              distant = 0.23
+            ) {
+              //  初始 角度
+              var center = modelViewerVariants.getBoundingBoxCenter().toString()
+              modelViewerVariants.cameraTarget = center
+
+              const startAngle = -((total_images - 1) / 2) * ddcap_range
+              const angles = {}
+
+              for (let i = 0; i < total_images; i++) {
+                angles[i + 1] = startAngle + i * ddcap_range
+              }
+              console.log(angles)
+
               let frames = []
 
               modelViewerVariants.removeAttribute('camera-controls')
 
-              for (let i = 0; i < totalImages; i++) {
-                modelViewerVariants.cameraOrbit = `${currentAngle}deg ${initialCameraOrbit[1]} ${initialCameraOrbit[2]}`
+              for (let i = 0; i < total_images; i++) {
+                setCameraOrbit(modelViewerVariants, distant, angles, i + 1)
+
+                // modelViewerVariants.cameraOrbit = `${currentAngle}deg ${initialCameraOrbit[1]} ${initialCameraOrbit[2]}`
                 await sleep(1000)
-                console.log(`Capturing image at angle: ${currentAngle}deg`)
+                // console.log(`Capturing image at angle: ${currentAngle}deg`)
                 let file = await captureImage(false)
                 frames.push(file)
-                currentAngle += angleIncrement
+                // currentAngle += angleIncrement
               }
               await sleep(1000)
               // 恢复到初始旋转角度
-              modelViewerVariants.cameraOrbit = initialCameraOrbit.join(' ')
+              // modelViewerVariants.cameraOrbit = initialCameraOrbit.join(' ')
               modelViewerVariants.setAttribute('camera-controls', '')
               return frames
             }
-            ddCap.addEventListener('click', async e => {
-              const angleIncrement = Number(ddcap_step.value),
-                totalImages = Number(total_images.value)
 
-              let images = await captureImages(angleIncrement, totalImages)
-              // console.log(images)
+            ddCap.addEventListener('click', async e => {
+              const distant = Number(ddcap_distant.value), //  23m
+                totalImages = Number(total_images.value),
+                angleIncrement = Number(ddcap_range.value)
+              console.log(angleIncrement, totalImages)
+              let images = await captureImages(
+                angleIncrement,
+                totalImages,
+                distant
+              )
+
               let dd = getLocalData(key)
               dd[that.id].images = images
               setLocalDataOfWin(key, dd)
             })
 
-            ddcap_range.addEventListener('input', async e => {
-              // console.log(ddcap_range.value)
+            ddcap_distant.addEventListener('input', async e => {
+              // console.log(ddcap_distant.value)
+              const center = modelViewerVariants.getBoundingBoxCenter().toString()
+              modelViewerVariants.cameraTarget = center;
               const initialCameraOrbit =
                 modelViewerVariants.cameraOrbit.split(' ')
-              modelViewerVariants.cameraOrbit = `${ddcap_range.value}deg ${initialCameraOrbit[1]} ${initialCameraOrbit[2]}`
+              modelViewerVariants.cameraOrbit = `${initialCameraOrbit[2]} ${initialCameraOrbit[1]} ${ddcap_distant.value}m`
               modelViewerVariants.setAttribute('camera-controls', '')
             })
 
-            ddcap_range_top.addEventListener('input', async e => {
-              // console.log(ddcap_range.value)
-              const initialCameraOrbit =
-                modelViewerVariants.cameraOrbit.split(' ')
-              modelViewerVariants.cameraOrbit = `${initialCameraOrbit[0]} ${ddcap_range_top.value}deg ${initialCameraOrbit[2]}`
-              modelViewerVariants.setAttribute('camera-controls', '')
-            })
+            // ddcap_range_top.addEventListener('input', async e => {
+            //   // console.log(ddcap_range.value)
+            //   const initialCameraOrbit =
+            //     modelViewerVariants.cameraOrbit.split(' ')
+            //   modelViewerVariants.cameraOrbit = `${initialCameraOrbit[0]} ${ddcap_range_top.value}deg ${initialCameraOrbit[2]}`
+            //   modelViewerVariants.setAttribute('camera-controls', '')
+            // })
 
             if (modelViewerVariants) {
               modelViewerVariants.style.width = `${that.size[0] - 48}px`
@@ -579,7 +610,7 @@ app.registerExtension({
               )
 
               // 更新尺寸
-              let w = that.size[0] - 48,
+              let w = that.size[0] - 128,
                 h = (w * bg_img.naturalHeight) / bg_img.naturalWidth
 
               if (modelViewerVariants) {
@@ -641,18 +672,25 @@ app.registerExtension({
 
           // 更新尺寸
           let dd = getLocalData('_mixlab_3d_image')
-          // console.log(dd[that.id],bg_url)
+
           if (dd[that.id]) {
             const { bg_w, bg_h } = dd[that.id]
-            if (bg_h && bg_w) {
-              let w = that.size[0] - 48,
-                h = (w * bg_h) / bg_w
+            let w = that.size[0] - 128
+            preview.style.width = `${w}px`
+            console.log('更新尺寸', w)
 
+            if (modelViewerVariants) {
+              modelViewerVariants.style.width = `${w}px`
+              modelViewerVariants.style.height = `${Math.round(
+                that.size[1] * 0.8
+              )}px`
+            }
+
+            if (bg_h && bg_w) {
+              let h = (w * bg_h) / bg_w
               if (modelViewerVariants) {
-                modelViewerVariants.style.width = `${w}px`
                 modelViewerVariants.style.height = `${h}px`
               }
-              preview.style.width = `${w}px`
             }
           }
 
@@ -735,7 +773,7 @@ app.registerExtension({
       // let base64 = await parseImage(url)
 
       let pre = widget.div.querySelector('.preview')
-      pre.style.width = `${node.size[0]}px`
+      pre.style.width = `${node.size[0] - 24}px`
       pre.innerHTML = `
         ${url ? `<img src="${url}" style="width:100%"/>` : ''}
         `

web/javascript/app_mixlab.js

@@ -4,7 +4,12 @@ import { api } from '../../../scripts/api.js'
 
 import { td_bg } from './td_background.js'
 // console.log('td_bg', td_bg)
-import { getUrl, base64Df, get_position_style, getObjectInfo } from './common.js'
+import {
+  getUrl,
+  base64Df,
+  get_position_style,
+  getObjectInfo
+} from './common.js'
 
 //本机安装的插件节点全集
 window._nodesAll = null
@@ -229,8 +234,6 @@ async function extractInputAndOutputData (
   return { input, output, seed, seedTitle }
 }
 
- 
-
 async function save_app (json) {
   let url = getUrl()
 
@@ -265,18 +268,20 @@ function downloadJsonFile (jsonData, fileName = 'mix_app.json') {
 async function save (json, download = false, showInfo = true) {
   let nodesAll = window._nodesAll || (await getObjectInfo())
 
-  console.log('####SAVE', nodesAll, json[0])
+  console.log('####SAVE', nodesAll, json)
 
   const name = json[0],
     version = json[5],
     share_prefix = json[6], //用于分享的功能扩展
     link = json[7], //用于创建界面上的跳转链接
     category = json[8] || '', //用于分类
+    idle_animation = json[9], //用于动画，比如数字人her
     description = json[4],
     inputIds = json[2].split('\n').filter(f => f),
     outputIds = json[3].split('\n').filter(f => f)
 
   const iconData = json[1][0]
+
   let { filename, subfolder, type } = iconData
   let iconUrl = api.apiURL(
     `/view?filename=${encodeURIComponent(
@@ -329,6 +334,27 @@ async function save (json, download = false, showInfo = true) {
     try {
       data.app.icon = await drawImageToCanvas(iconUrl)
     } catch (error) {}
+
+    let images = []
+    if (json[1].length > 1 && idle_animation) {
+      images = Array.from(json[1], j => {
+        let { filename, subfolder, type } = j
+        return api.apiURL(
+          `/view?filename=${encodeURIComponent(
+            filename
+          )}&type=${type}&subfolder=${subfolder}${app.getPreviewFormatParam()}${app.getRandParam()}`
+        )
+      })
+    }
+
+    try {
+      for (let index = 0; index < images.length; index++) {
+        const imgurl = images[index]
+        images[index] = await drawImageToCanvas(imgurl)
+      }
+      data.app.idle_animation = images
+    } catch (error) {}
+
     // console.log(data.app)
     // let http_workflow = app.graph.serialize()
     await save_app(data)
@@ -338,13 +364,17 @@ async function save (json, download = false, showInfo = true) {
 
     if (showInfo) {
       let open = window.confirm(
-        `You can now access the standalone application on a new page!\n${getUrl()}/mixlab/app?filename=${encodeURIComponent(
+        `You can now access the standalone application on a new page!\n${getUrl()}/mixlab/app${
+          data.app.idle_animation ? '/her.html' : ''
+        }?filename=${encodeURIComponent(
           data.app.filename
         )}&category=${encodeURIComponent(data.app.category)}`
       )
       if (open)
         window.open(
-          `${getUrl()}/mixlab/app?filename=${encodeURIComponent(
+          `${getUrl()}/mixlab/app${
+            data.app.idle_animation ? '/her.html' : ''
+          }?filename=${encodeURIComponent(
             data.app.filename
           )}&category=${encodeURIComponent(data.app.category)}`
         )

web/javascript/checkVersion_mixlab.js

@@ -3,7 +3,7 @@ import { app } from '../../../scripts/app.js'
 const repoOwner = 'shadowcz007' // 替换为仓库的所有者
 const repoName = 'comfyui-mixlab-nodes' // 替换为仓库的名称
 
-const version = 'v0.40.0'
+const version = 'v0.41.0'
 
 fetch(`https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`)
   .then(response => response.json())

web/javascript/image_mixlab.js

@@ -349,7 +349,7 @@ app.registerExtension({
           draw (ctx, node, widget_width, y, widget_height) {
             Object.assign(
               this.div.style,
-              get_position_style(ctx, widget_width, 44, node.size[1], 36)
+              get_position_style(ctx, widget_width, 44, node.size[1], 60)
             )
           }
         }
@@ -733,10 +733,10 @@ app.registerExtension({
           type: 'div',
           name: 'image_base64',
           draw (ctx, node, widget_width, y, widget_height) {
-            Object.assign(
-              this.div.style,
-              get_position_style(ctx, widget_width, 44, node.size[1], 44)
-            )
+            Object.assign(this.div.style, {
+              ...get_position_style(ctx, widget_width, y, node.size[1], 72),
+              top: `${widget_height}px`
+            })
           },
           serialize: false
         }
@@ -857,7 +857,7 @@ app.registerExtension({
       let imagePreview = node.widgets.filter(w => w.name == 'image_base64')[0]
       // console.log('#LoadImagesToBatch', imagesWidget.value?.base64)
       let imagesDiv = imagePreview.div.querySelector('.images_preview')
-
+      imagesDiv.innerHTML = ''
       for (const d of imagesWidget.value?.base64 || []) {
         let im = createInputImageForBatch(d, imagesWidget)
         imagesDiv.appendChild(im)
@@ -873,7 +873,7 @@ app.registerExtension({
         let imagePreview = node.widgets.filter(w => w.name == 'image_base64')[0]
 
         let imagesDiv = imagePreview?.div?.querySelector('.images_preview')
-
+        imagesDiv.innerHTML = ''
         for (const d of imagesWidget.value?.base64 || []) {
           let im = createInputImageForBatch(d, imagesWidget)
           imagesDiv.appendChild(im)

web/javascript/main_mixlab.js

@@ -1147,10 +1147,10 @@ app.registerExtension({
           type: 'video',
           name: 'FloatingVideo',
           draw (ctx, node, widget_width, y, widget_height) {
-            Object.assign(
-              this.card.style,
-              get_position_style(ctx, widget_width, y, node.size[1], 0)
-            )
+            Object.assign(this.card.style, {
+              ...get_position_style(ctx, widget_width, y, node.size[1], 40),
+              top: `${widget_height}px`
+            })
           }
         }
 

web/javascript/ui_mixlab.js

@@ -1342,7 +1342,9 @@ app.registerExtension({
 
     LGraphCanvas.prototype.text2text = async function (node) {
       let widget = node.widgets.filter(
-        w => w.name === 'text' && typeof w.value == 'string'
+        w =>
+          (w.name === 'text' || w.name === 'prompt') &&
+          typeof w.value == 'string'
       )[0]
       if (widget) {
         app.canvas.centerOnNode(node)
@@ -1653,7 +1655,9 @@ app.registerExtension({
 
       if (node.widgets) {
         let text_widget = node.widgets.filter(
-          w => w.name === 'text' && typeof w.value == 'string'
+          w =>
+            (w.name === 'text' || w.name === 'prompt') &&
+            typeof w.value == 'string'
         )
 
         let text_input = node.inputs?.filter(

web/javascript/video_mixlab.js

@@ -5,46 +5,13 @@ import { ComfyWidgets } from '../../../scripts/widgets.js'
 import { $el } from '../../../scripts/ui.js'
 import { injectCSS } from './common.js'
 
+import { get_position_style } from './common.js'
+
 injectCSS(`
 .hidden{
   display:none !important
 }`)
 
-function get_position_style (ctx, widget_width, y, node_height) {
-  const MARGIN = 4 // the margin around the html element
-
-  /* Create a transform that deals with all the scrolling and zooming */
-  const elRect = ctx.canvas.getBoundingClientRect()
-  const transform = new DOMMatrix()
-    .scaleSelf(
-      elRect.width / ctx.canvas.width,
-      elRect.height / ctx.canvas.height
-    )
-    .multiplySelf(ctx.getTransform())
-    .translateSelf(MARGIN, MARGIN + y)
-
-  return {
-    transformOrigin: '0 0',
-    transform: transform,
-    left:
-      document.querySelector('.comfy-menu').style.display === 'none'
-        ? `60px`
-        : `0`,
-    top: `0`,
-    cursor: 'pointer',
-    position: 'absolute',
-    maxWidth: `${widget_width - MARGIN * 2}px`,
-    // maxHeight: `${node_height - MARGIN * 2}px`, // we're assuming we have the whole height of the node
-    width: `${widget_width - MARGIN * 2}px`,
-    // height: `${node_height * 0.3 - MARGIN * 2}px`,
-    // background: '#EEEEEE',
-    display: 'flex',
-    flexDirection: 'column',
-    // alignItems: 'center',
-    justifyContent: 'space-around'
-  }
-}
-
 function videoUpload (node, inputName, inputData, app) {
   const imageWidget = node.widgets.find(w => w.name === 'video')
   let uploadWidget
@@ -53,13 +20,15 @@ function videoUpload (node, inputName, inputData, app) {
     type: 'div',
     name: 'upload-preview',
     draw (ctx, node, widget_width, y, widget_height) {
-      Object.assign(
-        this.div.style,
-        get_position_style(ctx, widget_width, 220, node.size[1]),
-        {
-          outline: '1px solid'
-        }
-      )
+      let d = {
+        ...get_position_style(ctx, widget_width - 20, 220, node.size[1], 72),
+        outline: '1px solid',
+        top: `${widget_height + 24}px`
+      }
+
+      delete d.height
+
+      Object.assign(this.div.style, d)
     }
   }
 
@@ -233,16 +202,17 @@ function offsetDOMWidget (widget, ctx, node, widgetWidth, widgetY, height) {
       elRect.height / ctx.canvas.height
     )
     .multiplySelf(ctx.getTransform())
-    .translateSelf(0, widgetY + margin)
+    .translateSelf(margin, widgetY + margin)
 
   const scale = new DOMMatrix().scaleSelf(transform.a, transform.d)
+
   Object.assign(widget.inputEl.style, {
     transformOrigin: '0 0',
     transform: scale,
-    left: `${transform.e}px`,
-    top: `${transform.d + transform.f}px`,
-    width: `${widgetWidth}px`,
-    height: `${(height || widget.parent?.inputHeight || 32) - margin}px`,
+    left: `${transform.a + transform.e + 56}px`,
+    top: `${transform.d + transform.f + 44}px`,
+    width: `${widgetWidth - 32}px`,
+    height: `${(height || widget.parent?.inputHeight || 32) - margin * 2}px`,
     position: 'absolute',
     background: !node.color ? '' : node.color,
     color: !node.color ? '' : 'white',
@@ -281,8 +251,21 @@ const createPreviewElement = (name, val, format) => {
     type,
     value: val,
     draw: function (ctx, node, widgetWidth, widgetY, height) {
-      const [cw, ch] = this.computeSize(widgetWidth)
-      offsetDOMWidget(this, ctx, node, widgetWidth, widgetY, ch)
+      
+      Object.assign(
+        this.inputEl.style,
+        get_position_style(ctx, widgetWidth - 12, 220, node.size[1], 44),
+        {
+          outline: '1px solid',
+          display: 'flex',
+          flexWrap: 'wrap',
+          flexDirection: 'row',
+          justifyContent: 'flex-start',
+          top: `${height + 24}px`
+        }
+      )
+
+
     },
     computeSize: function (_) {
       const ratio = this.inputRatio || 1
@@ -325,13 +308,14 @@ app.registerExtension({
           draw (ctx, node, widget_width, y, widget_height) {
             Object.assign(
               this.div.style,
-              get_position_style(ctx, widget_width, 188, node.size[1]),
+              get_position_style(ctx, widget_width - 12, 220, node.size[1], 44),
               {
                 outline: '1px solid',
                 display: 'flex',
                 flexWrap: 'wrap',
                 flexDirection: 'row',
-                justifyContent: 'flex-start'
+                justifyContent: 'flex-start',
+                top: `${widget_height + 24}px`
               }
             )
           }

web/javascript/visualization.js

@@ -85,7 +85,7 @@ function createVisualizer (node, inputName, typeName, inputData, app) {
         .translateSelf(margin, margin + widgetY)
 
       Object.assign(this.visualizer.style, {
-        left: `${transform.a * margin + transform.e}px`,
+        left: `${transform.a * margin + transform.e + 40}px`,
         top: `${transform.d + transform.f + top_offset}px`,
         width: `${w * transform.a}px`,
         height: `${
@@ -188,6 +188,7 @@ function registerVisualizer (nodeType, nodeData, nodeClassName, typeName) {
 
     nodeType.prototype.onExecuted = async function (message) {
       // Check if reference image and depth map are available
+      console.log("#message",message)
       if (message.reference_image && message.depth_map) {
         const params = {}
         params.reference_image = message.reference_image[0]

web/javascript/watcher_mixlab.js

@@ -105,13 +105,14 @@ app.registerExtension({
         orig_nodeCreated?.apply(this, arguments)
 
         // 虚拟的widget，用于更新节点，让其每次都运行
-        const widget = {
-          type: 'div',
-          name: 'seed',
-          draw (ctx, node, widget_width, y, widget_height) {}
-        }
+        // 改成了seed输入
+        // const widget = {
+        //   type: 'div',
+        //   name: 'seed',
+        //   draw (ctx, node, widget_width, y, widget_height) {}
+        // }
 
-        this.addCustomWidget(widget)
+        // this.addCustomWidget(widget)
 
         const watcher = this.widgets.filter(w => w.name == 'watcher')[0]
 
@@ -156,18 +157,19 @@ app.registerExtension({
         this.serialize_widgets = true
       }
 
-      const onExecuted = nodeType.prototype.onExecuted
-      nodeType.prototype.onExecuted = function (message) {
-        onExecuted?.apply(this, arguments)
-        console.log(message)
-        try {
-          let seed = this.widgets.filter(w => w.name === 'seed')[0]
-          if (seed) {
-            if (!seed.value) seed.value = 0
-            seed.value += 1
-          }
-        } catch (error) {}
-      }
+      // 改成seed的输入
+      // const onExecuted = nodeType.prototype.onExecuted
+      // nodeType.prototype.onExecuted = function (message) {
+      //   onExecuted?.apply(this, arguments)
+      //   console.log(message)
+      //   try {
+      //     let seed = this.widgets.filter(w => w.name === 'seed')[0]
+      //     if (seed) {
+      //       if (!seed.value) seed.value = 0
+      //       seed.value += 1
+      //     }
+      //   } catch (error) {}
+      // }
     }
   },
   async loadedGraphNode (node, app) {
@@ -196,13 +198,13 @@ app.registerExtension({
         }
       }
 
-      try {
-        let seed = node.widgets.filter(w => w.name === 'seed')[0]
-        if (seed) {
-          if (!seed.value) seed.value = 0
-          seed.value += 1
-        }
-      } catch (error) {}
+      // try {
+      //   let seed = node.widgets.filter(w => w.name === 'seed')[0]
+      //   if (seed) {
+      //     if (!seed.value) seed.value = 0
+      //     seed.value += 1
+      //   }
+      // } catch (error) {}
     }
   }
 })

webApp/index.html

@@ -706,9 +706,10 @@
             background: none !important;
             color: black !important;
         }
+
         .active:hover {
             /* border-color: yellow !important; */
-            border:1px solid yellow
+            border: 1px solid yellow
         }
 
         .unactive {
@@ -716,7 +717,7 @@
             color: #4a4a4a !important;
         }
     </style>
-   
+
     <link href="/mixlab/app/lib/photoswipe.min.css" rel="stylesheet">
     <link href="/mixlab/app/lib/classic.min.css" rel="stylesheet">
     <script src="/mixlab/app/lib/pickr.min.js"></script>
@@ -783,7 +784,7 @@
         // console.log(Lightbox)
         import { api } from "/mixlab/app/javascript/api.js";
         import Command from '/mixlab/app/javascript/command.js'
- 
+
         const base64Df =
             'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAwAAAAMCAYAAABWdVznAAAAAXNSR0IArs4c6QAAALZJREFUKFOFkLERwjAQBPdbgBkInECGaMLUQDsE0AkRVRAYWqAByxldPPOWHwnw4OBGye1p50UDSoA+W2ABLPN7i+C5dyC6R/uiAUXRQCs0bXoNIu4QPQzAxDKxHoALOrZcqtiyR/T6CXw7+3IGHhkYcy6BOR2izwT8LptG8rbMiCRAUb+CQ6WzQVb0SNOi5Z2/nX35DRyb/ENazhpWKoGwrpD6nICp5c2qogc4of+c7QcrhgF4Aa/aoAFHiL+RAAAAAElFTkSuQmCC'
 
@@ -1981,6 +1982,7 @@
                         e.preventDefault()
                         let inp = document.createElement('input')
                         inp.type = 'file'
+                        inp.setAttribute('accept', "audio/*")
                         inp.style.display = 'none'
                         inp.addEventListener('change', async e => {
                             e.preventDefault()
@@ -2481,7 +2483,7 @@
                 console.log(error)
             }
             // ksamplerCard.appendChild(status);
-            ksamplerCard.appendChild(seeds);
+            if (seeds.innerHTML) ksamplerCard.appendChild(seeds);
 
             // 动态组合和批量组合
             const promptLab = document.createElement('details');
@@ -2558,7 +2560,7 @@
             workflowSubmitPannel.id = 'workflow_submit_pannel';
 
             workflowSubmitPannel.appendChild(inputDetailsCard);
-            workflowSubmitPannel.appendChild(ksamplerCard);
+            if (ksamplerCard.children.length > 0) workflowSubmitPannel.appendChild(ksamplerCard);
             workflowSubmitPannel.appendChild(submitDiv);
 
             document.body.appendChild(workflowSubmitPannel);

webApp/javascript/command.js

@@ -389,14 +389,30 @@ function base64ToBlob (base64) {
   return blob
 }
 
+function generateRandomString (length = 16) {
+  const array = new Uint8Array(length)
+  window.crypto.getRandomValues(array)
+  return Array.from(array, byte => byte.toString(16).padStart(2, '0')).join('')
+}
+// window.crypto.subtle 在https协议下才能访问
 async function calculateImageHash (blob) {
-  const buffer = await blob.arrayBuffer()
-  const hashBuffer = await crypto.subtle.digest('SHA-256', buffer)
-  const hashArray = Array.from(new Uint8Array(hashBuffer))
-  const hashHex = hashArray
-    .map(byte => byte.toString(16).padStart(2, '0'))
-    .join('')
-  return hashHex
+  if (window.crypto && window.crypto.subtle && window.crypto.subtle.digest) {
+    try {
+      const buffer = await blob.arrayBuffer()
+      const hashBuffer = await crypto.subtle.digest('SHA-256', buffer)
+      const hashArray = Array.from(new Uint8Array(hashBuffer))
+      const hashHex = hashArray
+        .map(byte => byte.toString(16).padStart(2, '0'))
+        .join('')
+      return hashHex
+    } catch (error) {
+      console.error('Error computing hash with Web Crypto API:', error)
+      return generateRandomString()
+    }
+  } else {
+    console.error('Web Crypto API is not supported in this browser.')
+    return generateRandomString()
+  }
 }
 
 // 获取 rembg 模型
@@ -661,6 +677,47 @@ const _textNodes = [
   _colorNodes = ['Color'],
   _audioNodes = ['LoadAndCombinedAudio_']
 
+async function createVideoFromBase64Images (base64Images, frameRate = 24) {
+  const canvas = document.createElement('canvas')
+  const ctx = canvas.getContext('2d')
+
+  // Load the first image to set the canvas dimensions
+  const firstImage = new Image()
+  firstImage.src = base64Images[0]
+  await new Promise(resolve => (firstImage.onload = resolve))
+
+  canvas.width = firstImage.width
+  canvas.height = firstImage.height
+
+  const stream = canvas.captureStream(frameRate)
+  const recorder = new MediaRecorder(stream)
+  const chunks = []
+
+  recorder.ondataavailable = event => {
+    if (event.data.size > 0) {
+      chunks.push(event.data)
+    }
+  }
+
+  recorder.start()
+
+  for (const base64Image of base64Images) {
+    const img = new Image()
+    img.src = base64Image
+    await new Promise(resolve => (img.onload = resolve))
+
+    ctx.drawImage(img, 0, 0, canvas.width, canvas.height)
+    await new Promise(resolve => setTimeout(resolve, 1000 / frameRate))
+  }
+
+  recorder.stop()
+
+  await new Promise(resolve => (recorder.onstop = resolve))
+
+  const videoBlob = new Blob(chunks, { type: 'video/webm' })
+  return URL.createObjectURL(videoBlob)
+}
+
 export default {
   get_url,
   get_my_app,
@@ -693,5 +750,7 @@ export default {
   _slideNodes,
   _imageNodes,
   _colorNodes,
-  _audioNodes
+  _audioNodes,
+
+  createVideoFromBase64Images //把图片的base64转为video src使用
 }

webApp/lib/threeVisualizer.js

@@ -1,8 +1,15 @@
 import * as THREE from './three/three.module.js'
-import { api } from '../../../scripts/api.js'
+// import { api } from '../../../scripts/api.js'
 import { OrbitControls } from './three/OrbitControls.js'
 import { RoomEnvironment } from './three/RoomEnvironment.js'
 
+const api = {
+  apiURL: url => {
+    console.log(url)
+    return url
+  }
+}
+
 const visualizer = document.getElementById('visualizer')
 const container = document.getElementById('container')
 const progressDialog = document.getElementById('progress-dialog')
@@ -97,6 +104,7 @@ async function main (referenceImageParams, depthMapParams) {
   let imageHeight = 10 // Default height, will be updated based on the image's aspect ratio
   //   console.log('#referenceImageParams', referenceImageParams)
   if (referenceImageParams?.filename) {
+    console.log('referenceImageParams', referenceImageParams)
     const referenceImageUrl = api
       .apiURL('/view?' + new URLSearchParams(referenceImageParams))
       .replace(/extensions.*\//, '')
@@ -228,8 +236,8 @@ const sleep = (t = 1000) => {
 
 // 方法：旋转摄像机并拍摄图片 // 每次旋转的角度增量，转换为弧度
 async function captureImages (
-  totalFrames = 40,
-  angleIncrement = THREE.MathUtils.degToRad(0.5)
+  totalFrames = 20,
+  angleIncrement = THREE.MathUtils.degToRad(1.5)
 ) {
   // 计算场景中所有物体的中心点
   const box = new THREE.Box3().setFromObject(scene)
@@ -253,7 +261,8 @@ async function captureImages (
   )
 
   // 起始角度为从当前角度往左旋转 20 度的位置
-  const startAngle = initialAngle - (angleIncrement * totalFrames) / 2
+  const startAngle = initialAngle
+  //  - (angleIncrement * totalFrames) / 2
 
   for (let i = 0; i < totalFrames; i++) {
     const angle = startAngle + i * angleIncrement