From 9e252d92ef2459291ff4d052889acf3c1a6ec94b Mon Sep 17 00:00:00 2001 From: aven Date: Thu, 12 Jun 2025 17:00:51 +0800 Subject: [PATCH] =?UTF-8?q?1.=20=E6=8F=90=E5=8D=87=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E9=80=9F=E5=BA=A6=EF=BC=8C2.=20=E4=BC=98=E5=8C=96=E5=B7=A5?= =?UTF-8?q?=E4=BD=9C=E6=B5=81=EF=BC=8C3.=20=E5=AD=97=E5=B9=95=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E4=BF=9D=E5=AD=98=E4=B8=BAutf-8=E7=BC=96=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../av-audio2subtitle-with-prompt-1.json | 423 +++++++++++++++++ .../av-audio2subtitle-with-prompt-2.json | 440 ++++++++++++++++++ .../av-audio2subtitle-with-prompt.json | 311 ------------- .../av-audio2subtitle-without-prompt.json | 241 ---------- nodes.py | 54 ++- 5 files changed, 900 insertions(+), 569 deletions(-) create mode 100644 example_workflow/av-audio2subtitle-with-prompt-1.json create mode 100644 example_workflow/av-audio2subtitle-with-prompt-2.json delete mode 100644 example_workflow/av-audio2subtitle-with-prompt.json delete mode 100644 example_workflow/av-audio2subtitle-without-prompt.json diff --git a/example_workflow/av-audio2subtitle-with-prompt-1.json b/example_workflow/av-audio2subtitle-with-prompt-1.json new file mode 100644 index 0000000..1a11c48 --- /dev/null +++ b/example_workflow/av-audio2subtitle-with-prompt-1.json @@ -0,0 +1,423 @@ +{ + "id": "377e0192-2ad2-408a-871c-6a86cf058f73", + "revision": 0, + "last_node_id": 45, + "last_link_id": 77, + "nodes": [ + { + "id": 43, + "type": "easy showAnything", + "pos": [ + 737.1539306640625, + 51.4323844909668 + ], + "size": [ + 478.719970703125, + 307.6799621582031 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "anything", + "shape": 7, + "type": "*", + "link": 76 + } + ], + "outputs": [ + { + "name": "output", + "type": "*", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-easy-use", + "ver": "1.2.9", + "Node name for S&R": "easy showAnything" + }, + "widgets_values": [ + "{\"key\": \"737fb4ed-6c4a-4e66-b8ce-c2c0c416f45a\", \"text\": \"三 分 钟 吉 祥 q 弹 面 条 裹 满 浓 郁 汤 汁 每 一 口 都 香 到 上 头 无 论 是 深 夜 加 班 追 剧 嘴 馋 还 是 懒 人 素 食 x 泡 面 都 能 瞬 间 满 足 你 的 胃 经 典 红 烧 酸 辣 劲 爽 香 浓 豚 骨 多 种 口 味 总 有 一 款 让 你 欲 罢 不 能\", \"timestamp\": [[150, 350], [350, 530], [530, 710], [710, 870], [870, 1110], [1410, 1650], [1650, 1850], [1850, 2010], [2010, 2250], [2550, 2730], [2730, 2930], [2930, 3090], [3090, 3210], [3210, 3390], [3390, 3630], [3930, 4110], [4110, 4290], [4290, 4530], [4530, 4670], [4670, 4830], [4830, 5010], [5010, 5170], [5170, 5410], [5890, 6070], [6070, 6250], [6250, 6450], [6450, 6610], [6610, 6810], [6810, 6910], [6910, 7150], [7410, 7630], [7630, 7770], [7770, 7930], [7930, 8170], [8470, 8650], [8650, 8850], [8850, 8950], [8950, 9130], [9130, 9310], [9310, 9550], [9870, 10110], [10210, 10370], [10370, 10610], [10630, 10730], [10730, 10930], [10930, 11090], [11090, 11290], [11290, 11450], [11450, 11570], [11570, 11750], [11750, 11870], [11870, 12110], [12610, 12830], [12830, 13030], [13030, 13190], [13190, 13430], [13770, 14010], [14030, 14230], [14230, 14370], [14370, 14610], [14910, 15090], [15090, 15330], [15350, 15570], [15570, 15810], [16050, 16230], [16230, 16450], [16450, 16650], [16650, 16890], [17130, 17310], [17310, 17490], [17490, 17570], [17570, 17810], [17830, 17970], [17970, 18110], [18110, 18270], [18270, 18410], [18410, 18570], [18570, 19045]]}" + ] + }, + { + "id": 42, + "type": "easy showAnything", + "pos": [ + 1252.9384765625, + -201.37600708007812 + ], + "size": [ + 544.5440673828125, + 575.8159790039062 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "anything", + "shape": 7, + "type": "*", + "link": 73 + } + ], + "outputs": [ + { + "name": "output", + "type": "*", + "links": [ + 77 + ] + } + ], + "properties": { + "cnr_id": "comfyui-easy-use", + "ver": "1.2.9", + "Node name for S&R": "easy showAnything" + }, + "widgets_values": [ + "1\n00:00:00,150 --> 00:00:00,710\n3分钟即享\n\n2\n00:00:00,710 --> 00:00:03,210\nQ弹面条 裹满浓郁\n\n3\n00:00:03,210 --> 00:00:05,410\n汤汁 每一口都香到上头\n\n4\n00:00:05,890 --> 00:00:08,170\n无论是深夜加班、追剧嘴馋\n\n5\n00:00:08,470 --> 00:00:10,730\n还是懒人速食 X泡面都\n\n6\n00:00:10,730 --> 00:00:11,870\n能瞬间满足你的胃\n\n7\n00:00:11,870 --> 00:00:15,330\n经典红烧、酸辣劲爽、香浓\n\n8\n00:00:15,350 --> 00:00:17,810\n豚骨 多种口味 总有一款\n\n9\n00:00:17,830 --> 00:00:19,045\n让你欲罢不能\n\n" + ] + }, + { + "id": 38, + "type": "AVASRTimestamp", + "pos": [ + 343.9599304199219, + -215.4831085205078 + ], + "size": [ + 270, + 122 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 74 + } + ], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [ + 75 + ] + }, + { + "name": "json_result", + "type": "STRING", + "links": [ + 76 + ] + }, + { + "name": "asr_result", + "type": "ASRRESULT", + "links": [ + 71 + ] + } + ], + "properties": { + "aux_id": "ComfyUI-AV-FunASR.git", + "ver": "6840820f8d34ffb273e8776b8ba73a8fb0e89d17", + "Node name for S&R": "AVASRTimestamp" + }, + "widgets_values": [ + 300, + false + ] + }, + { + "id": 41, + "type": "AVFormat2Subtitle", + "pos": [ + 773.8641967773438, + -325.087890625 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "asr_result", + "type": "ASRRESULT", + "link": 71 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 72 + } + ], + "outputs": [ + { + "name": "subtitle", + "type": "STRING", + "links": [ + 73 + ] + } + ], + "properties": { + "aux_id": "ComfyUI-AV-FunASR.git", + "ver": "6840820f8d34ffb273e8776b8ba73a8fb0e89d17", + "Node name for S&R": "AVFormat2Subtitle" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 2, + "type": "LoadAudio", + "pos": [ + 51.75614929199219, + -214.3138885498047 + ], + "size": [ + 270, + 136 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 74 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "22_jidaohuman-megatts3_0.flac", + null, + null + ] + }, + { + "id": 39, + "type": "easy showAnything", + "pos": [ + 734.82470703125, + -204.00804138183594 + ], + "size": [ + 489.3119812011719, + 201.248046875 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "anything", + "shape": 7, + "type": "*", + "link": 75 + } + ], + "outputs": [ + { + "name": "output", + "type": "*", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui-easy-use", + "ver": "1.2.9", + "Node name for S&R": "easy showAnything" + }, + "widgets_values": [ + "三 分 钟 吉 祥 q 弹 面 条 裹 满 浓 郁 汤 汁 每 一 口 都 香 到 上 头 无 论 是 深 夜 加 班 追 剧 嘴 馋 还 是 懒 人 素 食 x 泡 面 都 能 瞬 间 满 足 你 的 胃 经 典 红 烧 酸 辣 劲 爽 香 浓 豚 骨 多 种 口 味 总 有 一 款 让 你 欲 罢 不 能" + ] + }, + { + "id": 44, + "type": "AVSaveSubtitles", + "pos": [ + 1844.3179931640625, + -202.0580291748047 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "subtitles", + "type": "STRING", + "widget": { + "name": "subtitles" + }, + "link": 77 + } + ], + "outputs": [], + "title": "保存字幕文件到output", + "properties": { + "aux_id": "ComfyUI-AV-FunASR.git", + "ver": "6840820f8d34ffb273e8776b8ba73a8fb0e89d17", + "Node name for S&R": "AVSaveSubtitles" + }, + "widgets_values": [ + "", + "subtitles" + ] + }, + { + "id": 25, + "type": "PrimitiveStringMultiline", + "pos": [ + 346.7919006347656, + -10.424053192138672 + ], + "size": [ + 263.8999938964844, + 168.8079833984375 + ], + "flags": { + "collapsed": false + }, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 72 + ] + } + ], + "title": "校准文字（如果没有请删除此结点）", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "PrimitiveStringMultiline" + }, + "widgets_values": [ + "3分钟即享，Q弹面条，裹满浓郁汤汁，每一口都香到上头！无论是深夜加班、追剧嘴馋，还是懒人速食，X泡面都能瞬间满足你的胃！经典红烧、酸辣劲爽、香浓豚骨，多种口味，总有一款让你欲罢不能。" + ] + }, + { + "id": 45, + "type": "Note", + "pos": [ + 118.52088165283203, + 204.437744140625 + ], + "size": [ + 574.4384155273438, + 157.96160888671875 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "原理介绍：https://mp.weixin.qq.com/s/ilGpgS5i_ZbE3SdA6godyw\n\nBy Aven，公众号：智效笔谈" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 71, + 38, + 2, + 41, + 0, + "ASRRESULT" + ], + [ + 72, + 25, + 0, + 41, + 1, + "STRING" + ], + [ + 73, + 41, + 0, + 42, + 0, + "*" + ], + [ + 74, + 2, + 0, + 38, + 0, + "AUDIO" + ], + [ + 75, + 38, + 0, + 39, + 0, + "*" + ], + [ + 76, + 38, + 1, + 43, + 0, + "*" + ], + [ + 77, + 42, + 0, + 44, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "frontendVersion": "1.18.9", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflow/av-audio2subtitle-with-prompt-2.json b/example_workflow/av-audio2subtitle-with-prompt-2.json new file mode 100644 index 0000000..0e527cc --- /dev/null +++ b/example_workflow/av-audio2subtitle-with-prompt-2.json @@ -0,0 +1,440 @@ +{ + "id": "377e0192-2ad2-408a-871c-6a86cf058f73", + "revision": 0, + "last_node_id": 46, + "last_link_id": 82, + "nodes": [ + { + "id": 43, + "type": "easy showAnything", + "pos": [ + 737.1539306640625, + 51.4323844909668 + ], + "size": [ + 478.719970703125, + 307.6799621582031 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "anything", + "shape": 7, + "type": "*", + "link": 80 + } + ], + "outputs": [ + { + "name": "output", + "type": "*", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-easy-use", + "ver": "1.2.9", + "Node name for S&R": "easy showAnything" + }, + "widgets_values": [ + "{\"key\": \"rand_key_gUe52RvEJgwBu\", \"text\": \"3 分 钟 即 享 q 弹 面 条 裹 满 浓 郁 汤 汁 每 一 口 都 香 到 上 头 无 论 是 深 夜 加 班 追 剧 嘴 馋 还 是 懒 人 速 食 x 泡 面 都 能 瞬 间 满 足 你 的 胃 经 典 红 烧 酸 辣 劲 爽 香 浓 豚 骨 多 种 口 味 总 有 一 款 让 你 欲 罢 不 能\", \"timestamp\": [[170, 330], [330, 510], [510, 670], [670, 910], [910, 1150], [1410, 1470], [1470, 1670], [1670, 1830], [1830, 2050], [2050, 2290], [2590, 2730], [2730, 2850], [2850, 2930], [2930, 3090], [3090, 3250], [3250, 3470], [3470, 3710], [3930, 4070], [4070, 4130], [4130, 4310], [4310, 4530], [4530, 4690], [4690, 4810], [4810, 4970], [4970, 5190], [5190, 5250], [5250, 5490], [5970, 6130], [6130, 6290], [6290, 6430], [6430, 6590], [6590, 6650], [6650, 6830], [6830, 7010], [7010, 7250], [7450, 7630], [7630, 7790], [7790, 8010], [8010, 8070], [8070, 8310], [8550, 8670], [8670, 8830], [8830, 8950], [8950, 9150], [9150, 9350], [9350, 9430], [9430, 9670], [9910, 10150], [10230, 10410], [10410, 10610], [10610, 10730], [10730, 10790], [10790, 10970], [10970, 11130], [11130, 11290], [11290, 11470], [11470, 11590], [11590, 11690], [11690, 11770], [11770, 11950], [11950, 12190], [12690, 12890], [12890, 13070], [13070, 13310], [13310, 13370], [13370, 13610], [13910, 14070], [14070, 14230], [14230, 14470], [14470, 14710], [14990, 15170], [15170, 15230], [15230, 15430], [15430, 15670], [15670, 15910], [16129, 16309], [16309, 16490], [16490, 16550]]}" + ] + }, + { + "id": 42, + "type": "easy showAnything", + "pos": [ + 1252.9384765625, + -201.37600708007812 + ], + "size": [ + 544.5440673828125, + 575.8159790039062 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "anything", + "shape": 7, + "type": "*", + "link": 73 + } + ], + "outputs": [ + { + "name": "output", + "type": "*", + "links": [ + 77 + ] + } + ], + "properties": { + "cnr_id": "comfyui-easy-use", + "ver": "1.2.9", + "Node name for S&R": "easy showAnything" + }, + "widgets_values": [ + "1\n00:00:00,170 --> 00:00:00,910\n3分钟即享\n\n2\n00:00:00,910 --> 00:00:02,930\nQ弹面条 裹满浓郁\n\n3\n00:00:02,930 --> 00:00:04,810\n汤汁 每一口都香到上头\n\n4\n00:00:04,810 --> 00:00:07,010\n无论是深夜加班、追剧嘴馋\n\n5\n00:00:07,010 --> 00:00:09,150\n还是懒人速食 X泡面都\n\n6\n00:00:09,150 --> 00:00:10,730\n能瞬间满足你的胃\n\n7\n00:00:10,730 --> 00:00:12,890\n经典红烧、酸辣劲爽、香浓\n\n8\n00:00:12,890 --> 00:00:15,230\n豚骨 多种口味 总有一款\n\n9\n00:00:15,230 --> 00:00:16,550\n让你欲罢不能\n\n" + ] + }, + { + "id": 39, + "type": "easy showAnything", + "pos": [ + 734.82470703125, + -204.00804138183594 + ], + "size": [ + 489.3119812011719, + 201.248046875 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "anything", + "shape": 7, + "type": "*", + "link": 79 + } + ], + "outputs": [ + { + "name": "output", + "type": "*", + "links": [] + } + ], + "properties": { + "cnr_id": "comfyui-easy-use", + "ver": "1.2.9", + "Node name for S&R": "easy showAnything" + }, + "widgets_values": [ + "3 分 钟 即 享 q 弹 面 条 裹 满 浓 郁 汤 汁 每 一 口 都 香 到 上 头 无 论 是 深 夜 加 班 追 剧 嘴 馋 还 是 懒 人 速 食 x 泡 面 都 能 瞬 间 满 足 你 的 胃 经 典 红 烧 酸 辣 劲 爽 香 浓 豚 骨 多 种 口 味 总 有 一 款 让 你 欲 罢 不 能" + ] + }, + { + "id": 44, + "type": "AVSaveSubtitles", + "pos": [ + 1844.3179931640625, + -202.0580291748047 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "subtitles", + "type": "STRING", + "widget": { + "name": "subtitles" + }, + "link": 77 + } + ], + "outputs": [], + "title": "保存字幕文件到output", + "properties": { + "aux_id": "ComfyUI-AV-FunASR.git", + "ver": "6840820f8d34ffb273e8776b8ba73a8fb0e89d17", + "Node name for S&R": "AVSaveSubtitles" + }, + "widgets_values": [ + "", + "subtitles" + ] + }, + { + "id": 2, + "type": "LoadAudio", + "pos": [ + 51.75614929199219, + -214.3138885498047 + ], + "size": [ + 270, + 136 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 78 + ] + } + ], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "22_jidaohuman-megatts3_0.flac", + null, + null + ] + }, + { + "id": 41, + "type": "AVFormat2Subtitle", + "pos": [ + 741.9202880859375, + -337.8655090332031 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "asr_result", + "type": "ASRRESULT", + "link": 81 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 72 + } + ], + "outputs": [ + { + "name": "subtitle", + "type": "STRING", + "links": [ + 73 + ] + } + ], + "properties": { + "aux_id": "ComfyUI-AV-FunASR.git", + "ver": "6840820f8d34ffb273e8776b8ba73a8fb0e89d17", + "Node name for S&R": "AVFormat2Subtitle" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 45, + "type": "AVSpeechTimestamp", + "pos": [ + 350.81390380859375, + -202.5904083251953 + ], + "size": [ + 270, + 122 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 78 + }, + { + "name": "text", + "type": "STRING", + "widget": { + "name": "text" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [ + 79 + ] + }, + { + "name": "json_result", + "type": "STRING", + "links": [ + 80 + ] + }, + { + "name": "asr_result", + "type": "ASRRESULT", + "links": [ + 81 + ] + } + ], + "properties": { + "aux_id": "ComfyUI-AV-FunASR.git", + "ver": "6840820f8d34ffb273e8776b8ba73a8fb0e89d17", + "Node name for S&R": "AVSpeechTimestamp" + }, + "widgets_values": [ + "", + false + ] + }, + { + "id": 25, + "type": "PrimitiveStringMultiline", + "pos": [ + 58.23122024536133, + -22.136831283569336 + ], + "size": [ + 259.0320129394531, + 168.8079833984375 + ], + "flags": { + "collapsed": false + }, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 72, + 82 + ] + } + ], + "title": "校准文字", + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.32", + "Node name for S&R": "PrimitiveStringMultiline" + }, + "widgets_values": [ + "3分钟即享，Q弹面条，裹满浓郁汤汁，每一口都香到上头！无论是深夜加班、追剧嘴馋，还是懒人速食，X泡面都能瞬间满足你的胃！经典红烧、酸辣劲爽、香浓豚骨，多种口味，总有一款让你欲罢不能。" + ] + }, + { + "id": 46, + "type": "Note", + "pos": [ + 81.41957092285156, + 204.1630096435547 + ], + "size": [ + 574.4384155273438, + 157.96160888671875 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": {}, + "widgets_values": [ + "原理介绍：https://mp.weixin.qq.com/s/ilGpgS5i_ZbE3SdA6godyw\n\nBy Aven，公众号：智效笔谈" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 72, + 25, + 0, + 41, + 1, + "STRING" + ], + [ + 73, + 41, + 0, + 42, + 0, + "*" + ], + [ + 77, + 42, + 0, + 44, + 0, + "STRING" + ], + [ + 78, + 2, + 0, + 45, + 0, + "AUDIO" + ], + [ + 79, + 45, + 0, + 39, + 0, + "*" + ], + [ + 80, + 45, + 1, + 43, + 0, + "*" + ], + [ + 81, + 45, + 2, + 41, + 0, + "ASRRESULT" + ], + [ + 82, + 25, + 0, + 45, + 1, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "frontendVersion": "1.18.9", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflow/av-audio2subtitle-with-prompt.json b/example_workflow/av-audio2subtitle-with-prompt.json deleted file mode 100644 index 01a43a2..0000000 --- a/example_workflow/av-audio2subtitle-with-prompt.json +++ /dev/null @@ -1,311 +0,0 @@ -{ - "id": "377e0192-2ad2-408a-871c-6a86cf058f73", - "revision": 0, - "last_node_id": 33, - "last_link_id": 50, - "nodes": [ - { - "id": 2, - "type": "LoadAudio", - "pos": [ - 182.18014526367188, - 389.2699890136719 - ], - "size": [ - 270, - 136 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "AUDIO", - "type": "AUDIO", - "links": [ - 24 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.32", - "Node name for S&R": "LoadAudio" - }, - "widgets_values": [ - "4_jidaohuman-megatts3_0.flac", - null, - null - ] - }, - { - "id": 25, - "type": "PrimitiveStringMultiline", - "pos": [ - 186.9199676513672, - 591.9599609375 - ], - "size": [ - 260, - 163 - ], - "flags": { - "collapsed": false - }, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "STRING", - "type": "STRING", - "links": [ - 47, - 48 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.32", - "Node name for S&R": "PrimitiveStringMultiline" - }, - "widgets_values": [ - "直观来说，测试时试错的步骤只有当能带来信息增益的情况下才有益，然而人们并没有在RL训练中告诉模型试错和反思带来的信息增益。" - ] - }, - { - "id": 18, - "type": "AVSpeechTimestamp", - "pos": [ - 489.56988525390625, - 374.6600341796875 - ], - "size": [ - 210, - 128 - ], - "flags": { - "collapsed": false - }, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "audio", - "type": "AUDIO", - "link": 24 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 47 - } - ], - "outputs": [ - { - "name": "text", - "type": "STRING", - "links": [] - }, - { - "name": "json_result", - "type": "STRING", - "links": [] - }, - { - "name": "asr_result", - "type": "ASRRESULT", - "links": [ - 43 - ] - } - ], - "properties": { - "Node name for S&R": "AVSpeechTimestamp" - }, - "widgets_values": [ - "", - false - ] - }, - { - "id": 30, - "type": "AVFormat2Subtitle", - "pos": [ - 518.5700073242188, - 561.3599853515625 - ], - "size": [ - 210, - 88 - ], - "flags": { - "collapsed": false - }, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "asr_result", - "type": "ASRRESULT", - "link": 43 - }, - { - "name": "text", - "type": "STRING", - "widget": { - "name": "text" - }, - "link": 48 - } - ], - "outputs": [ - { - "name": "subtitle", - "type": "STRING", - "links": [ - 45, - 50 - ] - } - ], - "properties": { - "Node name for S&R": "AVFormat2Subtitle" - }, - "widgets_values": [ - "" - ] - }, - { - "id": 31, - "type": "AVSaveSubtitles", - "pos": [ - 778.79541015625, - 357.02178955078125 - ], - "size": [ - 270, - 82 - ], - "flags": { - "collapsed": true - }, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "subtitles", - "type": "STRING", - "widget": { - "name": "subtitles" - }, - "link": 45 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "AVSaveSubtitles" - }, - "widgets_values": [ - "", - "subtitles" - ] - }, - { - "id": 33, - "type": "Bjornulf_ShowText", - "pos": [ - 814.5744018554688, - 427.3533935546875 - ], - "size": [ - 650.6199951171875, - 437.7900085449219 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "text_value", - "type": "STRING", - "link": 50 - } - ], - "outputs": [], - "properties": { - "cnr_id": "bjornulf_custom_nodes", - "ver": "1.0.0", - "Node name for S&R": "Bjornulf_ShowText" - }, - "widgets_values": [ - "1\n00:00:00,170 --> 00:00:02,330\n直观来说 测试时试错的步骤\n\n2\n00:00:02,330 --> 00:00:04,590\n只有当能带来信息增益的情况下才有益\n\n3\n00:00:04,590 --> 00:00:06,870\n然而人们并没有在RL训练\n\n4\n00:00:06,870 --> 00:00:09,330\n中告诉模型试错和反思带来的信息增益\n\n5\n00:00:09,330 --> 00:00:09,470\n\n\n" - ] - } - ], - "links": [ - [ - 24, - 2, - 0, - 18, - 0, - "AUDIO" - ], - [ - 43, - 18, - 2, - 30, - 0, - "ASRRESULT" - ], - [ - 45, - 30, - 0, - 31, - 0, - "STRING" - ], - [ - 47, - 25, - 0, - 18, - 1, - "STRING" - ], - [ - 48, - 25, - 0, - 30, - 1, - "STRING" - ], - [ - 50, - 30, - 0, - 33, - 0, - "STRING" - ] - ], - "groups": [], - "config": {}, - "extra": { - "frontendVersion": "1.18.9", - "VHS_latentpreview": false, - "VHS_latentpreviewrate": 0, - "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true - }, - "version": 0.4 -} \ No newline at end of file diff --git a/example_workflow/av-audio2subtitle-without-prompt.json b/example_workflow/av-audio2subtitle-without-prompt.json deleted file mode 100644 index ba40555..0000000 --- a/example_workflow/av-audio2subtitle-without-prompt.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "id": "377e0192-2ad2-408a-871c-6a86cf058f73", - "revision": 0, - "last_node_id": 35, - "last_link_id": 53, - "nodes": [ - { - "id": 2, - "type": "LoadAudio", - "pos": [ - 187.18014526367188, - 368.2699890136719 - ], - "size": [ - 270, - 136 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "AUDIO", - "type": "AUDIO", - "links": [ - 51 - ] - } - ], - "properties": { - "cnr_id": "comfy-core", - "ver": "0.3.32", - "Node name for S&R": "LoadAudio" - }, - "widgets_values": [ - "4_jidaohuman-megatts3_0.flac", - null, - null - ] - }, - { - "id": 31, - "type": "AVSaveSubtitles", - "pos": [ - 819.79541015625, - 213.02178955078125 - ], - "size": [ - 270, - 82 - ], - "flags": { - "collapsed": true - }, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "subtitles", - "type": "STRING", - "widget": { - "name": "subtitles" - }, - "link": 45 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "AVSaveSubtitles" - }, - "widgets_values": [ - "", - "subtitles" - ] - }, - { - "id": 34, - "type": "AVASRTimestamp", - "pos": [ - 467.7745361328125, - 206.14337158203125 - ], - "size": [ - 270, - 122 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "audio", - "type": "AUDIO", - "link": 51 - } - ], - "outputs": [ - { - "name": "text", - "type": "STRING", - "links": null - }, - { - "name": "json_result", - "type": "STRING", - "links": null - }, - { - "name": "asr_result", - "type": "ASRRESULT", - "links": [ - 52 - ] - } - ], - "properties": { - "Node name for S&R": "AVASRTimestamp" - }, - "widgets_values": [ - 300, - false - ] - }, - { - "id": 30, - "type": "AVFormat2Subtitle", - "pos": [ - 485.57000732421875, - 415.3599853515625 - ], - "size": [ - 210, - 88 - ], - "flags": { - "collapsed": false - }, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "asr_result", - "type": "ASRRESULT", - "link": 52 - } - ], - "outputs": [ - { - "name": "subtitle", - "type": "STRING", - "links": [ - 45, - 53 - ] - } - ], - "properties": { - "Node name for S&R": "AVFormat2Subtitle" - }, - "widgets_values": [ - "" - ] - }, - { - "id": 35, - "type": "Bjornulf_ShowText", - "pos": [ - 804.8345336914062, - 269.0433654785156 - ], - "size": [ - 398, - 387 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "text_value", - "type": "STRING", - "link": 53 - } - ], - "outputs": [], - "properties": { - "cnr_id": "bjornulf_custom_nodes", - "ver": "1.0.0", - "Node name for S&R": "Bjornulf_ShowText" - }, - "widgets_values": [ - "1\n00:00:00,150 --> 00:00:02,610\n直观来说测试时试错的步骤只有\n\n2\n00:00:02,610 --> 00:00:05,110\n当能带来信息增益的情况下才有益\n\n3\n00:00:05,410 --> 00:00:07,130\n然而人们并没有在all训练中\n\n4\n00:00:07,130 --> 00:00:10,225\n告诉模型试错和反思带来的信息增益\n\n" - ] - } - ], - "links": [ - [ - 45, - 30, - 0, - 31, - 0, - "STRING" - ], - [ - 51, - 2, - 0, - 34, - 0, - "AUDIO" - ], - [ - 52, - 34, - 2, - 30, - 0, - "ASRRESULT" - ], - [ - 53, - 30, - 0, - 35, - 0, - "STRING" - ] - ], - "groups": [], - "config": {}, - "extra": { - "frontendVersion": "1.18.9", - "VHS_latentpreview": false, - "VHS_latentpreviewrate": 0, - "VHS_MetadataImage": true, - "VHS_KeepIntermediate": true - }, - "version": 0.4 -} \ No newline at end of file diff --git a/nodes.py b/nodes.py index 66244d6..895fb19 100644 --- a/nodes.py +++ b/nodes.py @@ -6,6 +6,7 @@ import torchaudio import torchvision.utils as vutils import torch import json +import uuid from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks @@ -59,19 +60,31 @@ class AVSpeechTimestamp: if AVSpeechTimestamp.infer_ins_cache is None: model_root = os.path.join(folder_paths.models_dir, "ASR/FunASR") model_dir = os.path.join(model_root, name_maps_ms["fa-zh"]) + # vad_model = os.path.join(model_root, name_maps_ms["fsmn-vad"]) os.makedirs(model_dir, exist_ok=True) - AVSpeechTimestamp.infer_ins_cache = pipeline( - task=Tasks.speech_timestamp, + device = "cuda" if torch.cuda.is_available() else "cpu" + + AVSpeechTimestamp.infer_ins_cache = AutoModel( model=model_dir, model_revision="v2.0.4", - output_dir=temp_dir + device=device, # GPU加速 + disable_update=True ) - # save audio - audio_save_path = os.path.join(temp_dir, f"{int(time.time())}.wav") - torchaudio.save(audio_save_path, audio['waveform'].squeeze(0), audio["sample_rate"]) - rec_result = AVSpeechTimestamp.infer_ins_cache(input=(audio_save_path,text), data_type=("sound", "text")) + # save audio + uuidv4 = str(uuid.uuid4()) + audio_save_path = os.path.join(temp_dir, f"{uuidv4}.wav") + waveform = audio['waveform'] + sr = audio["sample_rate"] + waveform = torchaudio.functional.resample(waveform, sr, 16000) + torchaudio.save(audio_save_path, waveform.squeeze(0), 16000) + + rec_result = AVSpeechTimestamp.infer_ins_cache.generate( + input=(audio_save_path, text), + data_type=("sound", "text"), + ) + # print(rec_result) if rec_result: rec_result = rec_result[0] @@ -86,7 +99,7 @@ class AVSpeechTimestamp: # jr = json.dumps(rec_result, indent=4) text = rec_result.get("text") jr = json.dumps(rec_result, ensure_ascii=False) - print(text, jr) + # print(text, jr) return (text, jr, rec_result) @@ -122,28 +135,35 @@ class AVASRTimestamp: model=model_dir, vad_model=vad_model, punc_model=None, #"ct-punc", - device=device # GPU加速 + device=device, # GPU加速 + disable_update=True ) - # save audio - audio_save_path = os.path.join(temp_dir, f"{int(time.time())}.wav") - torchaudio.save(audio_save_path, audio['waveform'].squeeze(0), audio["sample_rate"]) + # save + uuidv4 = str(uuid.uuid4()) + audio_save_path = os.path.join(temp_dir, f"{uuidv4}.wav") + # 重新采样为16k + waveform = audio['waveform'] + sr = audio["sample_rate"] + waveform = torchaudio.functional.resample(waveform, sr, 16000) + torchaudio.save(audio_save_path, waveform.squeeze(0), 16000) rec_result = AVASRTimestamp.infer_ins_cache.generate(input=audio_save_path, batch_size_s=batch_size_s) + # print(rec_result) if rec_result: rec_result = rec_result[0] # infer if unload_model: import gc - if AVSpeechTimestamp.infer_ins_cache is not None: - AVSpeechTimestamp.infer_ins_cache = None + if AVASRTimestamp.infer_ins_cache is not None: + AVASRTimestamp.infer_ins_cache = None gc.collect() torch.cuda.empty_cache() - print("AVSpeechTimestamp memory cleanup successful") + print("AVASRTimestamp memory cleanup successful") # jr = json.dumps(rec_result, indent=4) text = rec_result.get("text") jr = json.dumps(rec_result, ensure_ascii=False) - print((text, jr, rec_result)) + # print((text, jr, rec_result)) return (text, jr, rec_result) @@ -197,7 +217,7 @@ class AVSaveSubtitles: def save_subtitles(self, subtitles, filename_prefix="subtitles"): filename = f"{filename_prefix}-{time.time()}.srt" save_path = os.path.join(self.output_dir, filename) - with open(save_path, "w") as f: + with open(save_path, "w", encoding='utf-8') as f: f.write(subtitles) return { "ui": { "text": save_path } }