From 8fec51cd5234019025dc83f79b5ca49e7dbc0cbe Mon Sep 17 00:00:00 2001 From: smthemex Date: Thu, 23 Oct 2025 14:22:57 +0800 Subject: [PATCH] init --- README.md | 13 ++- .../descript_quantize3.cpython-311.pyc | Bin 17061 -> 0 bytes .../codeclm/tokenizer/Flow1dVAE/model_1rvq.py | 42 ++++---- .../tokenizer/Flow1dVAE/model_septoken.py | 90 +++++++++++------- .../models/musicfm/musicfm_model.py | 48 +++++----- .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 227 bytes .../get_1dvae_large.cpython-311.pyc | Bin 0 -> 1234 bytes .../tools/__pycache__/mix.cpython-311.pyc | Bin 0 -> 3702 bytes .../__pycache__/torch_tools.cpython-311.pyc | Bin 0 -> 7888 bytes 9 files changed, 109 insertions(+), 84 deletions(-) delete mode 100644 SongGeneration/codeclm/tokenizer/Flow1dVAE/libs/rvq/__pycache__/descript_quantize3.cpython-311.pyc create mode 100644 SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/__init__.cpython-311.pyc create mode 100644 SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/get_1dvae_large.cpython-311.pyc create mode 100644 SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/mix.cpython-311.pyc create mode 100644 SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/torch_tools.cpython-311.pyc diff --git a/README.md b/README.md index 6327414..25e986a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [SongGeneration](https://github.com/tencent-ailab/SongGeneration):High-Quality Song Generation with Multi-Preference Alignment (SOTA),you can try VRAM>12G # Update +* 10/23 同步官方代码，删除fairseq库，已无安装难度； * 10/21同步官方代码，精简模型加载，删除hubert模型，优化lm模型加载顺序，避免转移到显存时峰值OOM； * 10/18 修改加载流程，支持最新的full ，new，large模型，large模型12GVram可能会OOM，修复高版本transformer 的函数错误/Modify the loading process to support the latest full, new, and large models, and fix function errors in higher versions of transformers * 07/29，支持bgm和人声（vocal，目前还是有bgm底噪）单独输出，选择mixed为合成全部，模型加载方式更合理，去掉诸多debug打印，新增save_separate按钮，开启则保存三个音频（bgm，vocal，mixed）； @@ -16,9 +17,8 @@ git clone https://github.com/smthemex/ComfyUI_SongGeneration.git ``` # 2. Requirements -* window平台最难装的就是[fairseq](https://github.com/facebookresearch/fairseq)库，python3.11的建议用轮子安装[liyaodev/fairseq](https://github.com/liyaodev/fairseq/releases/tag/v0.12.3.1)； + * 如果缺失库，打开requirements_orgin.txt文件，看是少了哪个，手动安装； -* The most difficult thing to install on the Windows platform is the Fairseq library. It is recommended to install it on wheels for version 3.11 [liyaodev/fairseq](https://github.com/liyaodev/fairseq/releases/tag/v0.12.3.1)； * If the library is missing, open the ’requirements_orgin.txt‘ file and see which one is missing, then manually install it； ``` @@ -32,17 +32,16 @@ pip install -r requirements.txt * 3.1.4 download htdemucs.pth [tencent/SongGeneration](https://huggingface.co/tencent/SongGeneration/tree/main/third_party/demucs/ckpt) * 文件结构如下,修改了加载流程，原来的结构也能用： ``` --- ComfyUI/models/SongGeneration/ +-- ComfyUI/models/SongGeneration/ # 24.4G all 整个文件夹的大小 |-- htdemucs.pth #150M |--prompt.pt # 3M |--new_prompt.pt # 3M |--model_2.safetensors |--model_2_fixed.safetensors - |--new_model.pt # rename from model.pt - |--large_model.pt # rename from model.pt - |-- ckpt/ # 24.4G all 整个文件夹的大小 + |--new_model.pt # rename from model.pt #可选 + |--large_model.pt # rename from model.pt #可选 + |-- ckpt/ |--encode-s12k.pt # 3.68G - |--models--lengyue233--content-vec-best/ -- ComfyUI/models/vae/ |--autoencoder_music_1320k.ckpt ``` diff --git a/SongGeneration/codeclm/tokenizer/Flow1dVAE/libs/rvq/__pycache__/descript_quantize3.cpython-311.pyc b/SongGeneration/codeclm/tokenizer/Flow1dVAE/libs/rvq/__pycache__/descript_quantize3.cpython-311.pyc deleted file mode 100644 index 627bb93d1a1e73b6b19ca992bc3e9824ed79a388..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17061 zcmcJ0Yit`=y4Va~;!7eW(UdGo(a7>c^h2^_CwAngV#%`CvTWIkb62r1LvcnDB|fAX z$`6I=8fb#hhTC{iXTx2%SYE(Ql^b8IP~4(wvB1^6xWF~gKQqJtA_g#Ev@mY{PiYc# zu|L{=-x+c^LrJkW?H-b6&Yb6WUf-MlzPj2;LHOVgSEm{eP}G0Nmr|%ICcgV`G(~+$ zu@pR9KD3ES|$GErPD z>tfyTt7dDSdcLwkkML8&SUr?mrhTP%)(iD&D@NT0b?UioU%~vpfu9<#+bC+xKh`kT zIJSMPX)FMBnmPY;OR3ygo2E#4!`9{8cD8N-;NqItdVDTGM`LeomdkD9{0p#Oxz^HH z`sY{h@{N{}W3;3$VXUK$#G0lPQC zoJxffNj@IYbxPqu!HxUwBt&0QDQ&4JDy3~9MNN~9!G>unN~P&1^uMN!i*=RynjQ|C zdE6z%lt?JHGs(owZZ>FAEMb09P;4`|@F`$V4D-i-dA9fZnPhz8&im&Nb$6GN*CX>n zDj5ejSxzWrhOQiB;P~rjW64|H?6tmu>#^v# zaGk$7cOANm@X^^+Xl_27NI|VbowIio`v=1+lzcTn<$|Ap`*W9i?(%=ZKf06KDK>O3 z?S=@xlB-8{^@x_9@1P20SVafsK=^JLqR(I*Ickh%sWB5v!$otmX3oM|;Ae%Ojk8SH zGt>mlR((=EW@X`GjM-QR#8s>lVms@C*pZ=b(_>E9+G?fd8W%|=`H>>z;WgMKCj;Sp zd^X0#xkQSr3zM8+u8lB4G#-tGc_x))E`<5naBAw#9)aPx*<>$iG?hxt3cX!jlhM@F z{CH<18Sk1Y*6U(9ZuY=UGTMRSLXoaX!#Gi(&g;06o5%w>+BfaA*~;9PS;3Z8JJ$CEr0OD1RLXPIaMQnwDUQCJayz{v=+ z;Rr_{z+N#x4@|TKAa*k^q#tbhQM0>2Fb zvMCmUi%sx#I7E3cEK?kzP$U)>giuJpl_r za_^ATIV^V$OZH2${gP|@(-hG`SwZ{A>@$h#~8`p9WlygWBRA7>{j36{G= z4Ou2N#R2XIgvq1;@dJLJ2RuBJ;a)Vc;UsEe$(A%Z5`xLCIFAWm>no<$zVx zFU6S>?T*5}MSvOUC^9Ut0gf zzwXR92TTUX==rDefQluNOdv-N>^VChL4Jxyiu4NcRj`V0hKEuE3@uDAR2)^I¬~ z0PnC-ijCz`VLRY(`pB0mIC_|Py#4U znxKEB3cv?eon7!(kbq+TE@&W#5Z$HL4^p1`&tvyv+3tJE2g!`N&=6Q_*qd+IyL3ou z=#m?{?zuDO%m*)<0+09Rt}eCzZtq|3{d)h?{j0mhrt?zMdAaHQTGMd8X;{2GB3>PR zc2jD)CO2Knc%Ch@{L2!!s5u4XV;|0 zQMqw6Q@w7Zs=W^nW$SeBDl|Zns>wTXOGSH&YJxx*Z{n-;Za{NHvUH!(_}a zJarE*0j(Kaxhi^2i}uqS2?ruGLO4!C!pK2QYdjP1J38*6jP{M3!&QpMl@40?z3@;R zT#2UhC~+t@RY3dNq;9316o;a0QCw1Y#;NAw*;v?Ku{q_U3(%ji9U>sm9`A#Xvqy8I zPj1M&-jzJPvZq(H_x{L4N1@%DQ^?J!BTKWU%E<@K!{nUv9WX%9#P`BOaR3*b0L%+W z}cBjXMuUuv8NG6dy)FU!P><`ni*4AXdONT_5J7EN2!+eq?ETaf+= zvcEYAMF!!i)d)Ll0s;-}1h67HSWA#rvLXvG2PEfEdvg}#C&TeZqXSsLOlda^;IO!; zoW}c?p{6LdV(nNn9(^^iwltm7WP(yIKG`ZGlkwDUlvWn4TS*XUtFh&x4bB8=4oG-t z;+?6+(i<>=nxrs|Tqy}Tojg;@gBikz)85j%aqn`qrF{L<$UH7qrK`3$4Ha{Jvs!GG zs*HQ1vF!jm3K=A%?c|)zyqD$hMSz5boiF$E zsR@+K)TgEilQtBUYbwvGoT@m}F6c$W0%d$VjS?yetU2u{ zZS@Rq217`L0btb5cnKadlK$-5($)!7)VhWjyyFZAEzE(FOvgBr3NkFAiG(_i0%KS; z14=m1?ku>~VyFkzc)}mj3O)Yn|3FyqUg2hApxk0UUR3pC!4@7*JmDtzBbZY83{8!t zm}ir>6tgfNSLl>tj)!l9b^vrr{0XS6*h>m2z75+Fy@PXNK0!Fta7?KJt~E53Oc6DL zVx5R3!zmtjlV}ntBlq!b&li1Bbru>)OaLDvxnoySyomQyS@qZ|w#fifpyp(jyv zQ7pjuhZVfQR+M7`i_hp z7}IM1=NIo^%(hAHX4&1m=4SG4CRe-EFS;4Y-7UMj3pJjH!F%!S)NiMMGyUt>3eaP<#P zs=hhPX5atqh2LCQqJKU7czCH>Za%Q|FIF4=sret8pN0Rv^&eZsx)G^vM6MgjR26FK z)@t6$*SwXzCDrVaYxV&1?D9M~@h#&qWdF6yU?H&Mx5s{SEI0S-laEiX1rFo`2bRK8;Gi5hxZIZyyuH?RJl}L& zYC0)5oy=S)wC$4H4lU0}ZGEd3q_#_$e%bvNaQQ85a&zbMPPyl7zWMB0;9@>-Q3_m= z1D7&G1y5kj)0+3R=1h{OUG}u+!greRcCEI_J^fPqfZRSHdYM9NhZuZUZtY!bJ(X`g zwd(jskJNfuZoT}_U8rxB>v!eCu$@bX<$Wje^(Vyo6WGtYOR=TDTp5*5ToSK_pNLyPR)filxzCN0_ngq04`d@WpppbM3n3DAy&LW(mKiYM85&`XCDcL?n*v0_P8 zD8wcskZlHBL9~qgVQ5Wc0fL2iymU%+Q8O9gVzE%@YntDQ_YBFvY4vI%ec~}fmP+^< zBG5&aKI2@Oo?A!$oz^~jozlfaG;KY$PHB;i zw!Vw6#mLrfZC zYE=AJI3dcyZW?jo!X#0qoQEBSU5|jS1ne+ZLFdh0O@Re6-plkQK&Qm^zz<9|^L#kQ zgy-2P+Dkx;BaJs&hr;6mSceWC={$JoP?wI%fsA7c6BENIvX<%;NqA=F0_Vekr^wlC<<=+DfbSw6EF#X=*C zvbh9otOJapPKdf^N#QufU}T;6K*j=wLHZNB!9-FaZ1q>@jqihIRBv9nhb9VbjC*LP z(Po*#`&tasxR@klETE;KGqwo}Yx>j1HZq|wl4f>6fhMF0M#NBJ9*$_-9E^)gjZ(mO zK8VmlH3?kf4a*{Z&-6>v15ovTP4P)63YryD=Rt)I5nG9X%@{_tRs7L;JO&&AsJLcN zE*#qYgleXX&f~FUBpee?LZZ|t1Dz3N37`-$Fra>yMzTZNi>fow!;L48!2KR1zOWSG z4lL4oiS?f4p81|BO_w>|d)BAaS8#H_fu9;+md$XtXh5UHUPBYi5VY~LP_EEX)iy*7 zH^^a93*WXM%v5b__Ri9QmFCs(?_0&O zsOX)RywkFGdfjZOtWp9uj|@G*>WVW88hBvHz$D3sAy=^i!!^nAD1GSWq%xTo39%T0 z4N;uh9ToT()D7ARJE+>PNMo`Vcw9(1SrC;xU|-Bf%t!FmTd`VSb0ij_nebmB`U8f$ z)DLx(qh=GE)^pdkhp~TmMsjt?t`5NXjPRS$9)A4&g{|iR4T6V_0z~G+zf^h~T9u1zanF=KcemL~stG!Nfu@THtAgpKX#_ ztb(6CZMs_pvQ4QKWz>H0f2BuiWvyqLF6+No-RYmGFs|u)CQ>AkCqYA|LzUB78ssV+ zB&zKBBO}4=$__}2j#P=h!d$j$1^sJC65%l-HjvI8ph0tfs*_;TCN<|c_G;Q(I|ZX@ zNVcL!Q{uE?uG$W51>uGjB>FscEnshc0t6|YE(GSUA!{R~{nU*WV&HTwx=mEtowgXu zufu9BRxj2pdKSHlwT3yB?dw~finL3=Mo%q9JzUy^*B<(>10!H{x{if42WB{s28OMNg4*NP`1A18%mH=^$ z6y2d-YXBCw6uL%zl=w(9?g|Vj0Dg)wxTGief3XFZ`q1NNFdBnQw4X`NsvcIFff6eZ zCp&QhW#Wj&T5!NJRO&Evcbq}Tk4_yc%y74RiT9J%4Je)nA6zz0s)MCC00!2J^q|!) zam^nwQGprd=ehl0v4te~^vx41GNhxEpl#w<16nVnIpA#DIVZ?QsvnjH-)a z67ygX<16L~eZoB1POZn+na|N&tyRXM`@k5i>ack5#{!o(VrMQpuFY_FgkGni$&Q5` zo7(WPJAEi0z}FjvjRTr0=o*{+Bttkh z+)>nPRYJCR6Vo`>|1+4!2^cr$x+-keI*2N~39!MR!Ic5Kt#oI=yd4BKbxVjf{dJHP zckfMn9f*oowUUCW`b@*gS_O|C`|B@HMTNK`{){!dbwR!3LF|J_l){v{fvi z3>ElG@UB$htwJpevfqSPR!ra@mc}yBsltr~Us{g8jqjF;`B;oUfJs|;b{6Dx(At8F z!3;lwjrU^oGmHrHeF5uP!F4P?t5{;-m>b-tGU=UIu7nCHcV zkEa=QvBWune;>1}iik*YBuZF@Pho*W!#?2F7Mcr1mD-Af0a@VRpxY0~0PjiCIV7PK z2Mcbr`bD5c1n$4im#KQ;Y7kwmg+Qwu*avD#)Sa$VcE{V#1I**z+}x8}OLI>a za&akeSPmRsPRU0GrNH2`HYso+Glc5Zc&;y(lJ^{0X_NLGTM7Su5=64;l^c@(g6zML z8GPOt$o4%R%=JC~*^*BVo|2kRNsXuF#?zVeg*}~13(JnB1$j?zW+>Z~4Q87p_s&9K z&u>rt=G0R6)1&I3nW4^=AFc;5Q~D6Xq(LHk?p&suYq+z5V_EK6+ehRht28V^Zzhcagh;Ayx^ zYS@!Ghu$E~BGbLxFEK~gT2JO%PfD$)<<`@Z`*h|~!M#13T((GW9bRkc&A0SQEyv}S z|itf&WyZ-ax`@`9{C3lmLqekK`^b-&h@5>%W%ozb1Ydl0Tdg`>#p;F}Xjs)<2i;pOg9pxnB@( z-je!niJn`c{g$du{5NnC4gQXrfOihm7DT`^fj|dB-Za!y1FI8cz|^h@AHcX)u=l|J zN6{Q!Hov8dAjwm>7zk+fBZ$JsR6O<106#kn)G{eF|+g6xetMWgvlixuEsxd$1sRQI*gBTug@l3Am zQ6l4dUh9AOu~geG*Mf1u>UfJpnZAOr@zJGRpX}Sc=IhD(dY0c_sU@Gjl{xZh_*=O7 z@5#RRG6My-=kuT4|JlQN$=xEmTSRxu3tj83ysvB7D*4`#eeXcsx@}qOFZX560G9LA zeK7ziC6b#HeL=|=lzqXaw?)rE(SA@x2uM{oM+o=fV3!dBAO(Y$P1(-P(v#rC^@-ui zRjQrwDpg!&t6l@D(+OD(It`uVJ_9$ZvKPJ9XbbNndK3R9ovvxoroY^khyW1UuK*cX z(>h-ad{Z?kWz@Fl03Dbf<>=~f?!^RX&&JvS?b*`~&{B}})~Ju@RGi=~VctS_ z{`X*unyw8sw}M;KM&Y`bLb_t4MK_>IcM(;738+%1Cn{@+09BfwR``AxYL>M>mW?$D zO0|do&VW+q6J?}Y6h=0nRSkP>jdj8kz}uGaGx-BkXIaF1Gn`u-{k2&4%xnFy85J7* zI|uj@gTHSXl>cA&ccYs#vh)XvLxM@p_&=yOMe7^8Rad2=J0iDiZrEvt6IL{jI2LI= z{{f~C_;Z_T-l2(20GvWqv&Wp?50oe7jUzvXu!~!@nRTZYPM6RGK!98=VXYtpOZZ2 zWzYGH<)y#z(Ns2-yY*yoc?6`BeCrX(e^mA#U8$A*$JYF(^8Qosub+|kMgJ+)2c+Px z{@!eAaAsWK7E{+yCZJVFKp_GGTHlbRA05f|=W25!f3@xLh0GjewzNGCEse-6U282T z^DQS=ZmrJY=ULmczGv;v=J2z6M?N(owOo-~uG|~W3_Lutk-Qe%wVw~&AIdgpjQkrB zTyT2WKyd%&?Wf0<=U3aref`qDetBQN=;;^j{p2)(0dE3Um!HIF3Zp1Sg!4i3tLg~u z;csC2FEI*X^btm3jF6`W;Tm0?P~6tJO0;2#O})VUmss`$qj8KP5b50{ILyT878caQ zR)c}7^a1-5x=CPELp8M&0?n|-h2Y*o`|d*Pj&)Ci_0T#6(TjTjx)on+lz;nr6{hS| zV_@BZDJRv?oIR8CWzWn0-RmyQa#PK1>(!XTZp;0U&MKpx%e3`2cvz4}PADcef3PN> zwL52h;s*KB+Fcp#Z?g{2xq&A`>lCKmJXWtit`fSi#Nkhw@V`9VvsV_37>LNm1+80J zAzz`+{ncfNaKj;pr;3U7Q^7XHmbx*vY^4Iui-i<*vT; z+FAC!}p z35C9`G-$LI|7#eB|3;Jl5C{Y#@pCw6S#{%pwjuI2S$+shh?WWTKk$bXGdODy^%245 z>VhKa|D*l5>Iig_|1a-ChS3Fbx)!v6&xr+}OQ diff --git a/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_1rvq.py b/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_1rvq.py index 15eb6ba..6a20b7e 100644 --- a/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_1rvq.py +++ b/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_1rvq.py @@ -19,11 +19,11 @@ from .libs.rvq.descript_quantize3 import ResidualVectorQuantize import folder_paths from .models_gpt.models.gpt2_rope2_time_new_correct_mask_noncasual_reflow import GPT2Model from .models_gpt.models.gpt2_config import GPT2Config - +from .our_MERT_BESTRQ.mert_fairseq.models.musicfm.musicfm_model import MusicFMModel, MusicFMConfig from torch.cuda.amp import autocast -from .our_MERT_BESTRQ.test import load_model +# from .our_MERT_BESTRQ.test import load_model class HubertModelWithFinalProj(HubertModel): def __init__(self, config): @@ -272,6 +272,7 @@ class PromptCondAudioDiffusion(nn.Module): ssl_layer=None, uncondition=True, out_paint=False, + ssl_path='ckpt/encode-s12k.pt' ): super().__init__() @@ -294,30 +295,35 @@ class PromptCondAudioDiffusion(nn.Module): self.rsq48towav2vec = torchaudio.transforms.Resample(48000, 16000) # self.wav2vec = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True) # self.wav2vec_processor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True) - self.bestrq = load_model( - model_dir='codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq', - checkpoint_dir='ckpt/encode-s12k.pt', - ) + # self.bestrq = load_model( + # model_dir='codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq', + # checkpoint_dir='ckpt/encode-s12k.pt', + # ) + ssl_path=os.path.join(folder_paths.models_dir,"SongGeneration/ckpt/encode-s12k.pt") + self.bestrq = MusicFMModel(MusicFMConfig()) + bestrq_weights = torch.load(ssl_path, map_location='cpu',weights_only=False)["model"] + self.bestrq.load_state_dict(bestrq_weights, strict=False) + del bestrq_weights self.rsq48tobestrq = torchaudio.transforms.Resample(48000, 24000) self.rsq48tohubert = torchaudio.transforms.Resample(48000, 16000) for v in self.bestrq.parameters():v.requires_grad = False self.rvq_bestrq_emb = ResidualVectorQuantize(input_dim = 1024, n_codebooks = 1, codebook_size = 16_384, codebook_dim = 32, quantizer_dropout = 0.0, stale_tolerance=200) for v in self.rvq_bestrq_emb.parameters():v.requires_grad = False - + # self.hubert = HubertModelWithFinalProj.from_pretrained(os.path.join(folder_paths.models_dir,"SongGeneration/ckpt/models--lengyue233--content-vec-best/snapshots/c0b9ba13db21beaa4053faae94c102ebe326fd68")) # for v in self.hubert.parameters():v.requires_grad = False self.zero_cond_embedding1 = nn.Parameter(torch.randn(32*32,)) # self.xvecmodel = XVECModel() - config = GPT2Config(n_positions=1000,n_layer=39,n_head=30,n_embd=1200) - unet = GPT2Model(config) - mlp = nn.Sequential( - nn.Linear(1200, 1024), - nn.SiLU(), - nn.Linear(1024, 1024), - nn.SiLU(), - nn.Linear(1024, 768) - ) + # config = GPT2Config(n_positions=1000,n_layer=39,n_head=30,n_embd=1200) + # unet = GPT2Model(config) + # mlp = nn.Sequential( + # nn.Linear(1200, 1024), + # nn.SiLU(), + # nn.Linear(1024, 1024), + # nn.SiLU(), + # nn.Linear(1024, 768) + # ) self.set_from = "random" #self.cfm_wrapper = BASECFM(unet, mlp,self.ssl_layer) self.mask_emb = torch.nn.Embedding(3, 48) @@ -540,7 +546,7 @@ class PromptCondAudioDiffusion(nn.Module): input_audio_0 = self.preprocess_audio(input_audio_0) input_audio_1 = self.preprocess_audio(input_audio_1) - self.bestrq.eval() + #self.bestrq.eval() # bestrq_middle,bestrq_last = self.extract_bestrq_embeds(input_audios) # bestrq_middle = bestrq_middle.detach() @@ -577,7 +583,7 @@ class PromptCondAudioDiffusion(nn.Module): input_audio_0 = self.preprocess_audio(input_audio_0) input_audio_1 = self.preprocess_audio(input_audio_1) - self.bestrq.eval() + #self.bestrq.eval() # bestrq_middle,bestrq_last = self.extract_bestrq_embeds(input_audios) # bestrq_middle = bestrq_middle.detach() diff --git a/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_septoken.py b/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_septoken.py index 58aa941..e721275 100644 --- a/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_septoken.py +++ b/SongGeneration/codeclm/tokenizer/Flow1dVAE/model_septoken.py @@ -21,9 +21,9 @@ from .libs.rvq.descript_quantize3 import ResidualVectorQuantize from .models_gpt.models.gpt2_rope2_time_new_correct_mask_noncasual_reflow import GPT2Model from .models_gpt.models.gpt2_config import GPT2Config - +from .our_MERT_BESTRQ.mert_fairseq.models.musicfm.musicfm_model import MusicFMModel, MusicFMConfig from torch.cuda.amp import autocast -from .our_MERT_BESTRQ.test import load_model +# from .our_MERT_BESTRQ.test import load_model class HubertModelWithFinalProj(HubertModel): def __init__(self, config): @@ -147,41 +147,55 @@ class BASECFM(torch.nn.Module, ABC): mu (torch.Tensor): output of encoder shape: (batch_size, n_channels, mel_timesteps, n_feats) """ - t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] + #t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] + dt = t_span[1:] - t_span[:-1] + t = t_span[:-1] + B = x.shape[0] + + if guidance_scale > 1.0: + def double(z): + return torch.cat([z, z], 0) if z is not None else None + attention_mask = double(attention_mask) + + x_next = x.clone() noise = x.clone() - # I am storing this because I can later plot it by putting a debugger here and saving it to a file - # Or in future might add like a return_all_steps flag - sol = [] + for i in tqdm(range(len(dt))): + ti = t[i] - for step in tqdm(range(1, len(t_span))): - x[:,0:incontext_length,:] = (1 - (1 - self.sigma_min) * t) * noise[:,0:incontext_length,:] + t * incontext_x[:,0:incontext_length,:] - if(guidance_scale > 1.0): + x_next[:, :incontext_length] = ( + (1 - (1 - self.sigma_min) * ti) * noise[:, :incontext_length] + + ti * incontext_x[:, :incontext_length] + ) - model_input = torch.cat([ \ - torch.cat([latent_mask_input, latent_mask_input], 0), \ - torch.cat([incontext_x, incontext_x], 0), \ - torch.cat([torch.zeros_like(mu), mu], 0), \ - torch.cat([x, x], 0), \ - ], 2) - timestep=t.unsqueeze(-1).repeat(2) - - dphi_dt = self.estimator(inputs_embeds=model_input, attention_mask=attention_mask,time_step=timestep).last_hidden_state - dphi_dt_uncond, dhpi_dt_cond = dphi_dt.chunk(2,0) - dphi_dt = dphi_dt_uncond + guidance_scale * (dhpi_dt_cond - dphi_dt_uncond) - else: - model_input = torch.cat([latent_mask_input, incontext_x, mu, x], 2) - timestep=t.unsqueeze(-1) - dphi_dt = self.estimator(inputs_embeds=model_input, attention_mask=attention_mask,time_step=timestep).last_hidden_state + if guidance_scale > 1.0: + model_input = torch.cat([ + double(latent_mask_input), + double(incontext_x), + torch.cat([torch.zeros_like(mu), mu], 0), + double(x_next), + ], dim=2) + timestep = ti.expand(2 * B) - dphi_dt = dphi_dt[: ,:, -x.shape[2]:] - x = x + dt * dphi_dt - t = t + dt - sol.append(x) - if step < len(t_span) - 1: - dt = t_span[step + 1] - t + else: + model_input = torch.cat([ + latent_mask_input, incontext_x, mu, x_next + ], dim=2) + timestep = ti.expand(B) + + v = self.estimator(inputs_embeds=model_input, + attention_mask=attention_mask, + time_step=timestep).last_hidden_state + v = v[..., -x.shape[2]:] + + if guidance_scale > 1.0: + v_uncond, v_cond = v.chunk(2, 0) + v = v_uncond + guidance_scale * (v_cond - v_uncond) + + x_next = x_next + dt[i] * v + + return x_next - return sol[-1] def projection_loss(self,hidden_proj, bestrq_emb): bsz = hidden_proj.shape[0] @@ -243,6 +257,7 @@ class PromptCondAudioDiffusion(nn.Module): snr_gamma=None, uncondition=True, out_paint=False, + ssl_path='ckpt/encode-s12k.pt' ): super().__init__() @@ -263,10 +278,15 @@ class PromptCondAudioDiffusion(nn.Module): self.rsq48towav2vec = torchaudio.transforms.Resample(48000, 16000) # self.wav2vec = Wav2Vec2BertModel.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True) # self.wav2vec_processor = AutoFeatureExtractor.from_pretrained("facebook/w2v-bert-2.0", trust_remote_code=True) - self.bestrq = load_model( - model_dir='codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq', - checkpoint_dir='ckpt/encode-s12k.pt', - ) + # self.bestrq = load_model( + # model_dir='codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq', + # checkpoint_dir='ckpt/encode-s12k.pt', + # ) + ssl_path=os.path.join(folder_paths.models_dir,"SongGeneration/ckpt/encode-s12k.pt") + self.bestrq = MusicFMModel(MusicFMConfig()) + bestrq_weights = torch.load(ssl_path, map_location='cpu',weights_only=False) + self.bestrq.load_state_dict(bestrq_weights, strict=False) + del bestrq_weights self.rsq48tobestrq = torchaudio.transforms.Resample(48000, 24000) self.rsq48tohubert = torchaudio.transforms.Resample(48000, 16000) for v in self.bestrq.parameters():v.requires_grad = False diff --git a/SongGeneration/codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq/models/musicfm/musicfm_model.py b/SongGeneration/codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq/models/musicfm/musicfm_model.py index 17d895c..a3dcd89 100644 --- a/SongGeneration/codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq/models/musicfm/musicfm_model.py +++ b/SongGeneration/codeclm/tokenizer/Flow1dVAE/our_MERT_BESTRQ/mert_fairseq/models/musicfm/musicfm_model.py @@ -4,14 +4,14 @@ from .model.musicfm_25hz import MusicFM25Hz # import sys, os # sys.path.append(os.path.dirname(os.path.abspath(__file__))) # from model.musicfm_25hz import MusicFM25Hz -try: - from fairseq.fairseq.dataclass import FairseqDataclass - from fairseq.fairseq.models import BaseFairseqModel, register_model - from fairseq.fairseq.tasks.fairseq_task import FairseqTask -except: - from fairseq.dataclass import FairseqDataclass - from fairseq.models import BaseFairseqModel, register_model - from fairseq.tasks.fairseq_task import FairseqTask +# try: +# from fairseq.fairseq.dataclass import FairseqDataclass +# from fairseq.fairseq.models import BaseFairseqModel, register_model +# from fairseq.fairseq.tasks.fairseq_task import FairseqTask +# except: +# from fairseq.dataclass import FairseqDataclass +# from fairseq.models import BaseFairseqModel, register_model +# from fairseq.tasks.fairseq_task import FairseqTask from dataclasses import dataclass, field from typing import List, Tuple, Optional @@ -22,7 +22,7 @@ from logging import getLogger logger = getLogger(__name__) @dataclass -class MusicFMConfig(FairseqDataclass): +class MusicFMConfig: label_rate:int = field(default=25) num_codebooks:int = field(default=1) codebook_dim:int = field(default=16) @@ -45,9 +45,9 @@ class MusicFMConfig(FairseqDataclass): SAMPLE_RATE = 24_000 -@register_model("musicfm", dataclass=MusicFMConfig) -class MusicFMModel(BaseFairseqModel): - def __init__(self, cfg: MusicFMConfig, task_cfg: FairseqTask): +# @register_model("musicfm", dataclass=MusicFMConfig) +class MusicFMModel(torch.nn.Module): + def __init__(self, cfg: MusicFMConfig): super().__init__() self.cfg = cfg self.model = MusicFM25Hz( @@ -92,18 +92,18 @@ class MusicFMModel(BaseFairseqModel): result["hidden_emb"] = hidden_emb return result - @classmethod - def build_model(cls, cfg: MusicFMConfig, task: FairseqTask): - """Build a new model instance.""" + # @classmethod + # def build_model(cls, cfg: MusicFMConfig, task: FairseqTask): + # """Build a new model instance.""" - model = MusicFMModel(cfg, task.cfg) - import numpy as np - s = 0 - for param in model.parameters(): - s += np.product(param.size()) - print('# of parameters: '+str(s/1024.0/1024.0)) - return model + # model = MusicFMModel(cfg, task.cfg) + # import numpy as np + # s = 0 + # for param in model.parameters(): + # s += np.product(param.size()) + # print('# of parameters: '+str(s/1024.0/1024.0)) + # return model - def get_losses(self, result, batch): - return result['losses'] + # def get_losses(self, result, batch): + # return result['losses'] \ No newline at end of file diff --git a/SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/__init__.cpython-311.pyc b/SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1aeec5a5241ba635b8c2ddc795a52456fe735f6c GIT binary patch literal 227 zcmZ3^%ge<81mQn|GeGoX5CH>>P{wCAAY(d13PUi1CZpdGKS_~7656;g^cTderElMoO%+HHK<0b>uCg~CTyB))|~~4`=iXxiZ!k2 z!2?H4NaV-~F>vu>_%n)W;${<0#sfDL4}_C%7FsCqX7XIB9WjN2x&3~uR0z&*<~J!&z1hIx)|dhSeV)@%_Hn-lQK z@a;=jkmI0RQblWEZM2a*s!3e<9?N_?PF3pktce3bTLX`cwiiZj_g$1jK3SCB%F}2O z4nB=AG@fk5MCephO!21F8KWk1^~Coq!y2iQRgr{qQkALP-$p6fR665g^j@wKkBIG3^*--S$j_mpnJ>^0`PUEtz~A785>aFwG_+;*~8oBJ(~t5!*9pq;QX!b4>7v zE(=6I*_%}$E)>K*u0=%32A_-Y3=_wSlz7b-Tki-ttc^b`KK7iM%8O?Ya=F&DXa+uq zdb$wFZ-M%h=gvO0T#Lnwi)T!64NTi9a__a}mfl&cIBt83Ir_3N2AF62#aWB%Ir_%1 zblYIF)^NEJsRD~Ru|D7=IQri~)zOaHvt)0pgH3gC4`Jor%EHc#!R5+o=2LaKx|H6N z0gD%6^+&k;6Y2!JXm)ULa1s1FbA5SW<;i-`$mF*(`DP|xf4-9*S|8m`k2KRG^|6qs zKS}=&(!VmaT5Jy7-^w>g|28Q!NufasVX8OmzEYp?(?I=WUrzo;^2sfnPiDK=C4iW? z)KOPlHY1=VFB;=>B}VnK!FXjDs`ttkuL)N4I8_%xN0l2s_2&GeE1dWeEXXAe6kqU literal 0 HcmV?d00001 diff --git a/SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/mix.cpython-311.pyc b/SongGeneration/codeclm/tokenizer/Flow1dVAE/tools/__pycache__/mix.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9fcfe31391697f059761d6632920e1821301b02d GIT binary patch literal 3702 zcmahMTTC0-b;h0<+xUSWKtPyhnk6ojtO-yS5WxwM?gJ9S(ryteIdRdJ({)*2NlY86#c`vHnpt@fuqcl>}! zpuNDEd(Sw2pB*6_td37GX(KByeJJ-aq{X1Fu6mp1WN{p8Isko>Ko*Y ziq*0-eAO&-gP5UMa*A;1USgJszZV!k-1MjmGZ|_D;`OUMiN!?$mc~MP&XNMbs$!s8 zAPZks9Rsol!1<^wEv{B@mkcEYHLH36t4C!_oMI^fj2i?QEX^{kO2Ju{$Z8&d4Ufup z#Au*DgjS&5q{TKKEE%+G8z&OEU=f9Kzc4sjdc24;UoWyVu(>?6}>I6afZ~XNyTHn8p&wl=g zo3Y^{(eqQje@<>R-v9cK!@kCw|3J_0HM<oqqSE(^;C%dX^&5P;ib5vEE1tIX4yJd}qC!mv=|~A#V1yr3Yv|f#7U3 z^of`A$GrUPSRi!S$^O(e3b;@pG8^byVM(v;6kLZ$8@ap`J0ZXAC*VQ2p3;L(|$v8DFjfz15Ez2E>MeT{USR zd{IQ}&|B}IhJLZ3f0tI-2X+{uvE_c(ulllQa|@_r7#($qM_s7FwX0FvPm?>^!?tx4 zZ0{0O)fo&;XUN3ZcNGL!Ka*xM3^KNf#@~3 zL?J0kgVjVer8yKULdhvma_Y&6FVM>3HA1AwkF{|P6f0fP;-gV~zT#LYYY_-R13wf_ z#G9(R5&;1=K`~WojBB|+fy9cU#9u~nxh;+b9Om-K?6bLqgF>`ldztUx!UbJGzu%U_ zY{UiRH4AkLJ!PGOPDxidUA#i5P|(YL3fs&YqDS^@7vPSRC7(6K^q?DV?U5WIm=;?T z*DK^1N-_k!Qa^Z789T;9Vn?myQ?4L(n2fQcWB*ssDk({bhZW^nO(lAPQPM&TmiJj1dM;=F`fZi%=j+% zsXO3jo983V*Wx^mWw{~nI0hx^!uZrEkFW3Le_*`ChQ=yOEI}xoz~rz ziJ1@aL3dQ5c{k_tO6q`@o9EqEd3-ZG4grTzq9dUtj^(k_d39jjz$xN@;r2}GZdv+1kzKzUI(dL)+n><|afp{8@lc3!ld zPmXNY)vwiLsJr$xd)9*LewchO-+26+uE%}3)4$hlS~sUr-}_?U`%f>TzVoPkQf!|@ zjg!f9*gz(A9huuia~n42g|&Xm{-ba0-OudZxet-OU$pll>wst-*fI>{wT932xAZC3 zvT4PXG%cH6KyU3!jVuqX41t4S(Vt@PonHG%=0dh6*ZQy?*?UBL57PIF`rgDy-ekVr zwc3{+`C@2w2$@=-*Xpf_3BCrdOVOZkVU-)yFx=r`uiO0%1PZ5GnTE<#i&Og>*xn0S zA%pxsGH4cF(F!?xFJ84k3AEg%s5c8Hs}fXr4<|g#AenFz!rwRpmNPOQWW%gTh4&Sj z%I#ZVSQ6Iszr9;Dm;=)Yu*r9UK;Op@WwKVPzAAf6E`|k<;qyHtWH$NK z-)je<$)yqte`^M4KPd z?lO^-Wy9nWln#tLUExD`fL-uM&Sl{%AfR`Einf5SxC8Q|KX!!$L@Xe{xFRV0Plfv< z2nw|C%!(A3Rtg`kBksF1JMYZS&OGzH?>qWiueXMPlx==@@t=GI@$aZmO4MHFc@{EX z5&|KRF=EcTlXE04Q*#u)({nVwGjps!33SX6cg{IULSQ~4<~V@`=@NW`^D}a;M&O|2 z7F-}bf?ueCXI`N}@W8!R@WN9bG;5($Co~FmaIY7dgnGF91oA2oJoF3=YvvgmU63^C zcyuL9qDOnl0e2d%=YI|2OF|`-irU-dyh=QR{Nr+Y4dx+TtzDV{dmpHGoT~e<_b8BS zpta_<`W2`(mLOpK^vuGkdxO9K@#@6re^}i-9Sbrht0blHqRGU?Py#+Gv|`esTQYng z_hIM*F_M(xG8${(yVc24H_j*Hk=5CYe=;;w%H9a4WF;A&Pb39VE{BTHkZ+rrtC1Wy37}Gq?4=k;k-UJLI6a&?H z7?k9N@wniU&x6PiMgGvk?gzd3YkKQ|(K?{>1DR>`?8JkSU(9}Wy|AKp4;$UX`r#4d z@W|!`{qR|x9|sk;e?5G6RAXA6LpMaYJm*kg;KFP9d&qogTZQVmPg5%Si9-c>2lpUR zD$F)6qh1K9Mg`=m_?CN`i-syb2;i>@>CHj{Y=$Kc2br z)Y+&x+w!k$%xra>(>l(<8fN*Tt2sBiAtIm?`qb$<8Qt{l3Oe|wYEXsKVZUrx{o!8t+eq)+Q9Kz$m!QN;yJR$j!XaEs(nr!4& zC~?R1Dj{h>ECY$X6;gnUxWs%!3dKd)LZJIAx6ZU>CbGlp*NR+Yj?=k7X1wU+vO}MyGO3+f|Ghi8?tFcI zTCeRlYWufq2e)bmHyVF+M6Vq+YDcs5PF>@DSMEfUF(FU2m4nvPD--_U$hk zvyH{N#`O^T16=m!+6(pQPiq3bzAXFH$**^OqOLD*Ih!?S^S*9<4*i|ay^}k?9yRI$ zpz5y6T*jcZ;jwH}{s!amm`lBUPxx3FRUb#JYP) zt!5G`(bq|kgn}qyF9{5=r45}>kJx)J-A1d4lfYK-5_O$O&@1$H;zLqF5ZX(KSM5FK zE5=mP0xZ+j{i&4g&Di@u`nIR#u~&bcy*F%IDFIj?*;cRs6zm4)%(KiWfSWR1Nh!Jj z5EhpK%HoGOhj+?a)5J%J#KNg4u}l#XTI`_5tii@FloFyzlZhomf>{$u zCBmptC}wh!D2L)pF%erqlUWrbj|w3$~t%M1itl zXGx9dDJFADmh3cbdL_}K?|Gyy9S7Auxa5BY0fg;uyFZp6*8Rr}|FKUlXQs2acK8;9 z@7m`3xA^|T;*(duPHB9<&d(bBtoGgy4gOkYdY59@o*jQ6->mz44S#Qz%W~iDQWQMo zyc+*np;_mS8{Bb?JB~Q}R4$_XyA6Lg(gV$Jzun=R)>rcFIzMRegWLS6E&kNzkj|em z_%oKZ?|u;c#l%+^3yXT!8%Ea~dh4*!I;``vDFSU9Rl zgc9I;%_mZElL^P7OC}deDM_Saa_@ztXy{fUlH?S#>GLie0BJSQ^~PC8`THfo+u-7{u*#xfJdzQOIj zvs-;L42Dbw3b=x_e6QvUtkLH&9FW&uD6~u zTF;g1FeosEHC^cP*6iUcTNm<0M2y%1KXk|QH(-}$3UwwD-T}-=D z8)|h139uQ2<<|T~#YXT{euH!obj~6@1SS|Xf~>pU`hCMU+*>V=6f!`+Ewg&dtN~y4 z!)rQ!-QcfhroRb}Y#x0wr3Wt?!OJ?+lbL|#c}LMx|C1|UT*-~+mUT~u;pu>_(%Qe> z`o>o48=E9hZ0i}L^^C?G`lh3|aAKoh?>K98oc&#}$Q{nz{7;%XPdfgKKoWY(Ozd)m zr(W}Q=T|B=lY}@YeBh3H3N;7sCkR#!+6~}-rvY*K@@|6;$_ewY&D}@?^S!h#IQ9a) z=gtSJc4qa?sNA>E&JJG~BaJv$CGgH*X7)jYF%+aRUZhDFxrG#m1quXfX&Q+tcp%pR zgJaL>^;V+exO54sufQe$7(^91{>WwDbr4+R`rDs=ocVaCd!Ufoi0j=KjP47+ixxev z z*RA{3JJx;oQa}0Vi;wQ6@1=9g5Ha}jz6U9t@6x!gr+j@@vdl&WEC{q__&cXx@ZAC2 z44l%uzHkl!4jcGc)q%|ntlML>b~t^HiD3mt)mCr{&hK$= zsLDt|t@78adf`;3%0C77=gvbXo{sCB#=jgd1lS^Q92Mx2!W5%;Lo77G+A(D zmR2P+cGC&Y4vI}p2RhjbDR|j~E{mv#tlM!2J44t>A1ddLTO96?lc|I-WLo#*rW+0c zMTkDpQ`2h|_S(wG@wWCXNDFa|pn=at=3#{MB}M5bXhz_YzXSm;j<*#e&i;)~{nZn? z_oU%Hxv^|`N4C9VTi!9Rp(nrc>m!?s`p}dyG^IC88x7N$KP|eOa$e2dtufsg-_Ot-hRYy; zf*URj@(+M_k&#~@kY7*>ri>UnQZ6gYCL$MsoUs#c}$;8qf?I?~QW4QvdsQk4XhcB;+`#Lb&H1n1H&1)X&5CD`=b zK2<@D1egs7ExJ)g$@0zq5EE`wqyYjBqRDTnAga>Q7t2OVvuOO#wFWue` zlF`v!=_v}Ja00riveEYb(jL`g>#rR8+n%DMIIBO8y}iAJgKm#Ns~#M5L*FWkDFN&& zg5W+S1bGOY!PhfgQAL#SDYEGS*YH*}fzP95lLbd@L6qvCndyMMkc^v7c`+45=oO>842qb7M9`h#Nxi)Gv>yGI?^Ymm{bmWc*sEV6NiQuvfs zlA%5Umpll&2`pP@cV1c_$Xwp>AA0CAng{+~`SrxVbZ(0O*!$1D`VS_IA57@4Pa3aJ zeogAlZyC*R>HbTG{}SjkHJ{C_pVpax!2~oWu;Z-Bp4OeshO;?0vgPd3oL$AhF^y}3 z-_yXc!dxL*m@@*ysOWnQS<^l&?mDvd+3}r@lN+hcw|}+rr18n}Z=1h9p}qaizYS}% zbK0NZFlKLR^D(_6Zgj+Tu088of3wInY;%XV!0WuIb3F#vqj5b&u4$WV+v3`CcXY1T z;CeOuLk_X}@Z5u$!s*R={m7ffkvDbzw85W#>4So&@UMaQHFu}RbiQcU(MH6S_ur5K z|Al!m7=+DTMMrqkLLgU3hWKO{2yy@H76*LJG!TdC%$KPWUQgG-d?j2twIr7-W#bw;8yj&)$M0NC&w;){0z2xl2eOKNnM+wO(lU{*J(-!;}0ScRHRFQZaErC*Uatl59N4w`HQE+`344cWX) zltrtDtluT=I>hgdqoaiL$8mqM}{2+l#n1z>q%FY){m(WX~>P7j-_eYnQN# PUJrQ+4d_6D=k9+27v`Ub literal 0 HcmV?d00001