UhBdZddlZddlmZmZmZmZddlZddl Z ddl Z ddl m Z ddl m Z mZmZddlmZddlmZdd lmZdd lmZdd lmZmZdd lmZmZmZmZm Z dd l!m"Z"ddl#m$Z$m%Z%ddl&m'Z'm(Z(e%jRe*Z+dZ,de jZde.de.fdZ/ d{de jZde.dee jZfdZ0 d|dee.e.fde1de.dee jdde.dejff dZ4Gdd e jjZ6Gd!d"e jjZ7Gd#d$e jjZ8Gd%d&e jjZ9Gd'd(e jjZ:Gd)d*e jjZ;Gd+d,e jjjZ<Gd-d.e jjZ=Gd/d0e jjZ>Gd1d2e jjZ?Gd3d4e jjZ@Gd5d6e jjZAGd7d8e jjZBGd9d:e jjZCGd;de jjZEGd?d@e jjZFGdAdBe jjZGGdCdDe jjZHGdEdFe jjZIGdGdHe jjZJe$GdIdJe"ZKGdKdLeKZLGdMdNeKZMGdOdPeKZNGdQdReKZOGdSdTeKZPGdUdVeKZQGdWdXeKZRGdYdZeKZSGd[d\e jjZTGd]d^e jjZUe$d_`GdadbeKZVe$dc`GdddeeKeZW d}dfeKde jdgee jdee jddhe1die1dje1dkee jjdleYdmeYdee jee je jfffdnZZe$do`GdpdqeKZ[e$dr`GdsdteKZ\Gdudve jjZ]e$dw`Gdxdye"Z^gdzZ_y)~zPyTorch SpeechT5 model.N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossL1Loss)ACT2FN)GenerationMixin)is_deepspeed_zero3_enabled)is_fsdp_managed_module)_prepare_4d_attention_mask!_prepare_4d_causal_attention_mask)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutputSeq2SeqSpectrogramOutput)PreTrainedModel)auto_docstringlogging)SpeechT5ConfigSpeechT5HifiGanConfig input_ids pad_token_iddecoder_start_token_idc|j|j}|ddddfj|ddddf<||dddf<| td|j |dk(||S)z1 Shift input ids one token to the right. Nrrz1self.model.config.pad_token_id has to be defined.i) new_zerosshapeclone ValueError masked_fill_)rrrshifted_input_idss /var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/speecht5/modeling_speecht5.pyshift_tokens_rightr)4s}"++IOO<(CRC0668ae4adLMM""#4#AB%%&:f&DcJ  //r*r# mask_prob mask_length min_masksreturnc|\}dkr tdkDrtdddtjjdj fd}|-|j j djnt|Dcgc]}c}}tj|ft } g} |} | d k(r| S|D]} || } tjjtj| dz z | d }t|d k(rdz }n|d }tj|tj| | z tj |zg}| j#|tj$| } tj&| dddddf|| f} | j)|| z} tjddddf}tj&||| fj)|| z}| |z} | j+dz kDrdz | | dz kD<tj,| | dd| Scc}w) af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `ct|zz z}t|}|zkDrz}|dz z |krt|dz z d}|S)z;Given input length, compute how many spans should be maskedrr)intmax) input_lengthnum_masked_spanepsilonr4r3r5sequence_lengths r(compute_num_masked_spanz6_compute_mask_indices..compute_num_masked_spansoi,6DwNOoy9 [ (? :-r?s `` ` @@r(_compute_mask_indicesrbZs0#(JQABB_$]^i]j&&7q :  iinnQ$$&G  $  % ##B'..0',Z'8 9!o 9HHj/:$GM1/Ba%5 1,?II,, IIlkAo6 7RW-   !Q &-q0N.q1NNN (;o(MUWU]U] ^ao o p  !!"34/52"451a:&5H+(V,33J@SVa@abii $T4]3Goog 4G'UV^^'+5G,g5/A"55GVYZGZ-!0CCDm%7B? w:s$ I+c&eZdZdfd ZdZxZS)SpeechT5NoLayerNormConvLayercdt||dkDr|j|dz nd|_|j||_t j |j|j|j||j||j|_ t|j|_ y)Nrr kernel_sizestridebias)super__init__conv_dim in_conv_dim out_conv_dimrConv1d conv_kernel conv_stride conv_biasconvr feat_extract_activation activationselfconfiglayer_id __class__s r(rkz%SpeechT5NoLayerNormConvLayer.__init__s =CCEOOPQRUXUbUbcdUeeii338a@EEnVXY 1  !))S%++na"@AqIC  ""#C Q vve--/00r*rpast_key_values_lengthc|j\}}|j||j|j|j}|jdz|z}||j jdkDr4|j ||jz|j|j|j jd|jdj||djS)Nrrr!) size"create_position_ids_from_input_idsrrrrrrr index_selectrrH)rwrrbszseq_len position_idsmax_poss r(rz-SpeechT5SinusoidalPositionalEmbedding.forwardAs ~~' W>>y$JZJZ\rsvv    ""Q&0 T\\&&q) )   g 3T5G5GIYIY Z||((L,=,=b,ABGGWVXY``bbr*c|j|j}tj|dj ||z|z}|j |zS)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr)ner:rcumsumtype_aslong)rwrrrmaskincremental_indicess r(rzHSpeechT5SinusoidalPositionalEmbedding.create_position_ids_from_input_idsPsW||K(,,.$||Da8@@FI__cgg"'')K77r*r|r)rrr__doc__r:rrkr staticmethodrrno_gradTensorrrrrs@r(rrsNScS#SHUXMSG3GsGQYZ]Q^G1c1#1HUXM11"U]]_ c cs c cbc88478QYZ]Q^8r*rc$eZdZfdZdZxZS)SpeechT5PositionalConvEmbeddingct|tj|j|j|j |j dz|j |_tjj}ttjjdr$tjjj}tr(ddl }|jj|jj d5||jdd|_dddt|jdrU|jjj j"}|jjj j$}n,|jj&}|jj(}|jj+|||jj+||n||jdd|_t-|j |_t0|j2|_y#1swY'xYw) Nr)rgpaddinggroups weight_normr) modifier_rankweight)namerparametrizations)rjrkrro hidden_sizenum_conv_pos_embeddingsnum_conv_pos_embedding_groupsrsutilsrrrr deepspeedzeroGatheredParametersr original0 original1weight_gweight_vregister_external_parameterSpeechT5SamePadLayerrr rtru)rwrxrrrrrzs r(rkz(SpeechT5PositionalConvEmbedding.__init__cs II      6622a777   hh** 288,,m <((33??K % ' 224993C3CST2U I' aH  Ityy"459955<<FF9955<<FF99--99-- NN 6 6tX F NN 6 6tX F#DIIH!DDI+F,J,JK  !?!?@ I Is IIc|jdd}|j|}|j|}|j|}|jdd}|SNrr)rrsrrur}s r(rz'SpeechT5PositionalConvEmbedding.forwardsV%//15  -0  ]3  6 %//15 r*rrs@r(rrbsABr*rc*eZdZdZdfd ZdZxZS) SpeechT5ScaledPositionalEncodinguS Scaled positional encoding, see §3.2 in https://arxiv.org/abs/1809.08895 ctj||}tjd|jd}tjtjd|dtj j tjd|z z}tj|j |z|dddddf<tj|j |z|dddddf<|jd}t|1|jd|dtj| |_||_tjj%tj&d |_y) NrrrrAg@peFrp?)rrLrOrrrrrrrrrjrkrrDropoutdropoutr Parametertensoralpha)rwrrmax_lenrpositiondiv_termrzs r(rkz)SpeechT5ScaledPositionalEncoding.__init__s& [[# &<<7+55a899ell1c1EKKHNNPUYU]U]^eUfilUlSmmoii 08 ;<1add7 ii 08 ;<1add7 \\!_  T2%8zzG, XX'' S(9: r*c||j|jddd|jdfzz}|j|}|S)Nr)rrrr)rwrs r(rz(SpeechT5ScaledPositionalEncoding.forwardsBDJJMchhqkM)9!:::ll3 r*)i)rrrrrkrrrs@r(rrs ;r*rc&eZdZdfd ZdZxZS)"SpeechT5RelativePositionalEncodingct|||_||_tj j d|z||_yr)rjrkr max_lengthrr Embeddingpe_k)rwrrrzs r(rkz+SpeechT5RelativePositionalEncoding.__init__s8 $HH&&q:~s; r*c~|jd}tjd|j|jtj }|dddf|dddfz }|j |||j k<|j dz |||j k\<||j z}|j|S)NrrrrB)r#rrOrrrrr)rwr~rpos_seqs r(rz*SpeechT5RelativePositionalEncoding.forwards%%a(,,q'*--]5I5IQVQ[Q[-\!T'"WT1W%55/3.>4??**+.2oo.A4??*+DOO+yy!!r*)irrs@r(rrs < "r*rc$eZdZfdZdZxZS)rcPt||dzdk(rd|_yd|_y)Nrrr)rjrknum_pad_remove)rwrrzs r(rkzSpeechT5SamePadLayer.__init__s) #:Q#>!#Car*cV|jdkDr|ddddd|j f}|SNr)rr}s r(rzSpeechT5SamePadLayer.forwards6    ")!Q0F43F3F2F0F*FGMr*rrs@r(rrsKr*rc.eZdZdZfdZdZdZxZS)SpeechT5FeatureEncoderz.Construct the features from raw audio waveformc t||jdk(rDt|dgt |j dz Dcgc]}t ||dzc}z}nV|jdk(r.t |j Dcgc]}t||}}ntd|jdtj||_ d|_ d |_ ycc}wcc}w) Ngroupr)ryrlayerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)rjrkfeat_extract_normrrKnum_feat_extract_layersrdrr%r ModuleList conv_layersgradient_checkpointing_requires_grad)rwrxirrzs r(rkzSpeechT5FeatureEncoder.__init__s   # #w .5fqIJNSTZTrTruvTvNwNIJ,Va!eDNK % % 0HMfNlNlHmCD*6A>K01I1I0JJst ==5&+#"Ns C" C'cJ|jD] }d|_ d|_yNF) parameters requires_gradr)rwparams r(_freeze_parametersz)SpeechT5FeatureEncoder._freeze_parameterss(__& (E"'E  (#r*c |dddf}|jr|jrd|_|jD]K}|jr5|jr)|jr|j |j |}D||}M|SNT)rtrainingrrr_gradient_checkpointing_func__call__)rwr+r~ conv_layers r(rzSpeechT5FeatureEncoder.forwards$QW-    4==*.M '** :J""t'B'Bt}} $ A A''!! !+= 9  :r*)rrrrrkrrrrs@r(r r s8#&$ r*r c$eZdZfdZdZxZS)SpeechT5FeatureProjectionc4t|tj|jd|j |_tj|jd|j|_ tj|j|_ y)Nr!eps) rjrkrrrllayer_norm_epsrLinearr projectionrfeat_proj_dropoutrrwrxrzs r(rkz"SpeechT5FeatureProjection.__init__sf ,,vr':@U@UV))FOOB$79K9KLzz&":":; r*cp|j|}|j|}|j|}||fSr|)rr)r)rwr~norm_hidden_statess r(rz!SpeechT5FeatureProjection.forwards:!__];(:;  ]3 000r*rrs@r(r#r#s < 1r*r#c6eZdZfdZdZ d dej deejdeejfdZ de dejfdZ d e eje ffd Z d d ejdeejdeejfd ZxZS)SpeechT5SpeechEncoderPrenetct|||_t||_t ||_|jdkDs|jdkDrEtjtj|jj|_t!||_t%|j&|j(zdz|j|j(|_y)Nr0r)rjrkrxr feature_encoderr#feature_projectionmask_time_probmask_feature_probrrrrruniform_masked_spec_embedrpos_conv_embedrmax_speech_positionsrpos_sinusoidal_embedr+s r(rkz$SpeechT5SpeechEncoderPrenet.__init__s  5f=";F"C  3 &&*B*BS*H%'\\%,,v?Q?Q2R2[2[2]%^D "=fE$I  ' '&*=*= = A      % !r*c8|jjyr|)r1rrws r(freeze_feature_encoderz2SpeechT5SpeechEncoderPrenet.freeze_feature_encoders //1r*r+r-mask_time_indicesc|j|}|jdd}||j|jd|}|j |\}}|j |||}|j |}||z}| |jdj}n=tj|jddtj|j}|j|}||z}||fS)Nrr)r=r-r) r1r"_get_feature_vector_attention_maskr#r2_mask_hidden_statesr7rrrrLrr9) rwr+r-r=extract_featuresr~positional_conv_embedding padding_mask positional_sinusoidal_embeddingss r(rz#SpeechT5SpeechEncoderPrenet.forwards  // =+55a;  %!DD &&q)N +/*A*ABR*S' '00 ->~1 %)$7$7 $F!%(AA  %),,Q/446L ;;}':':2A'>ejjYfYmYmnL+/+D+D\+R(%(HH n,,r*feature_vector_lengthc|jddddf}|j|jtj}|j d}tj ||f|j|j}d|tj|j d|j|dz f<|jdgjdjdgj}|S)Nr!rrrrr) r _get_feat_extract_output_lengthsrrrr#rLrBrrOfliprM)rwrEr-non_padded_lengthsoutput_lengthsrYs r(r?z>SpeechT5SpeechEncoderPrenet._get_feature_vector_attention_mask=s,22r2:1b5A>>?QRUUV[V`V`a#))!,  . /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOr*r[cd}t|jj|jjD]\}}||||}|S)zH Computes the output length of the convolutional layers c>tj||z |ddzS)Nfloor) rounding_moder)rdiv)r<rgrhs r(_conv_out_lengthzVSpeechT5SpeechEncoderPrenet._get_feat_extract_output_lengths.._conv_out_lengthRs"99\K7wWZ[[ [r*)ziprxrprq)rwr[rQrgrhs r(rHz'> @W@W#X Q K,]KPM Qr*r~ct|jdds|S|j\}}}|)|jj |j ||<n|jj dkDr|jrt||f|jj |jj||jj}tj||jtj}|jj |j ||<|jjdkDr|jrt||f|jj|jj |jj"}tj||jtj}|dddfj%d|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://arxiv.org/abs/1904.08779). apply_spec_augmentTNr)r3r4r-r5r)r3r4r5r!)getattrrxrr6rrBr3rrbmask_time_lengthmask_time_min_masksrrrrMr4mask_feature_lengthmask_feature_min_masksexpand)rwr~r=r-rYr?rmask_feature_indicess r(r@z/SpeechT5SpeechEncoderPrenet._mask_hidden_states]st{{$8$? 4A3E3E3G0 O[  (/3/E/E/H/HI\I\/]M+ , [[ ' '! + 5_-++44 KK88-++99 ! !& ->}G[G[chcmcm n /3/E/E/H/HI\I\/]M+ , ;; ( (1 ,#8[)++77 KK;;++<< $ $)<<0D]MaMainisis#t #74#@#G#GO]_#` 23M. /r*NN)rrrrkr<rrr LongTensor FloatTensorrr:r?rrHr@rrs@r(r/r/s "2 6:9= -ll -!!1!12 -$E$5$56 -F  ]b]m]m  eEDTDTVYDY>Z &:>59 ,((,$E$5$56,!!1!12 ,r*r/cfeZdZfdZdZ ddej deej fdZxZ S)SpeechT5SpeechDecoderPrenetc Xt|||_tjt |j Dcgc]=}tj|dk(r |jn |j|j?c}|_ tj|j|j|_ t|j|j|j|_tj|j"|jz|j|_ycc}wr )rjrkrxrrrKspeech_decoder_prenet_layersr( num_mel_binsspeech_decoder_prenet_unitslayersr final_layerrpositional_dropoutr8encode_positionsspeaker_embedding_dimspeaker_embeds_layerrwrxrrzs r(rkz$SpeechT5SpeechDecoderPrenet.__init__s  mm vBBC     +,6F''v7Y7Y66   99V%G%GI[I[\ @  % %     ' '!  %'IIf.J.JVM_M_._agasas$t! sAD'ctj|d|}|jdj|j ddd}tj |dk(|ddzd|z z S)Nrrr)r bernoullirrepeatrwhere)rw inputs_embedsrr all_maskss r(_consistent_dropoutz/SpeechT5SpeechDecoderPrenet._consistent_dropoutsd}Q/15NN1%,,]-?-?-BAqI {{9>=!>LII  <<99A=!C   ..6 f::Q> > ggiDO"DOzz&"G"GH r*c|j|}|j|}|j|j|}|j|}|Sr|)rsrrurr}s r(rz"SpeechT5BatchNormConvLayer.forwardsJ -0  6 ?? & OOM:M ]3 r*rrrs@r(rzrzsI<r*rzc^eZdZfdZdej fdZdej fdZxZS)SpeechT5SpeechDecoderPostnetc t|||_tj|j |j |jz|_tj|j |j|_ tjt|jDcgc]}t||c}|_ycc}wr|)rjrkrxrr(rrcr,feat_outprob_outrrKr}rzrerks r(rkz%SpeechT5SpeechDecoderPostnet.__init__s   &"4"4f6I6IFLcLc6cd  &"4"4f6M6MN mm Ct{{<#>CCZP $Z__j1 (|((*5 //!$yyz/C/CAq/IJ    3#7'"J J6dnn8LgW^7_6`a %%'(*   $$//166sT^^7KRQUQ^Q^_iijkmnoI << =3J3J2r3RSL'11!Q7<<dnn$m&8&8&;]=O=OPQ=RL L (L  %""$a'(BB 7a'8R7SS\]k]p]p]r\st(,,S$..'7SVddL',,S4>>-A7GTL}},,\r,B  &##%$..):: Et~~FWEXY',,./1+//2q!<|?P?PQTVZVdVdfmov?wwL',,S4>>-A7GTL  %1$5$5c4>>7T[$\ !055cDNN6JGU\]L$( !]]**<4<AAr*)r0FT)NNNNNF)rrrrr:rrMrkrrrrrrrrs@r(rrSsF  CCC C  C  C6eU\\eCece 488<152604"'yB||yB#5<<0yB!u||!45 yB !. yB "%,,/ yB  -yB yB u||Xell3XeELL>Q5RR SyBr*rc$eZdZfdZdZxZS)SpeechT5FeedForwardct|tj|j|_tj |j||_t|jtrt|j|_ n|j|_ tj ||j|_tj|j|_yr|)rjrkrractivation_dropoutintermediate_dropoutr(rintermediate_dense isinstance hidden_actstrr intermediate_act_fn output_densehidden_dropoutoutput_dropout)rwrxintermediate_sizerzs r(rkzSpeechT5FeedForward.__init__s $&JJv/H/H$I!"$))F,>,>@Q"R f'' -'-f.?.?'@D $'-'8'8D $II&79K9KL jj)>)>?r*c|j|}|j|}|j|}|j|}|j |}|Sr|)rrrrrr}s r(rzSpeechT5FeedForward.forwardsX// > 00? 11-@ ))-8 ++M: r*rrs@r(rrs @r*rc eZdZdeffd Z d dej deej deej deej def dZ xZ S) SpeechT5EncoderLayerrxct|t|j|j|j d|_tj|j|_ tj|j|j|_ t||j|_tj|j|j|_y)NFrrrrr%)rjrkrrencoder_attention_headsattention_dropout attentionrrrrrr'rrencoder_ffn_dim feed_forwardfinal_layer_normr+s r(rkzSpeechT5EncoderLayer.__init__ s *((44,,   zz&"7"78 ,,v'9'9v?T?TU/8N8NO " V-?-?VEZEZ [r*r~r-rrrc|}|j|||||\}}}|j|}||z}|j|}||j|z}|j |}|f} |r| |fz } | S)as Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, hidden_size)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(config.encoder_attention_heads,)`. position_bias (`torch.FloatTensor`): relative position embeddings of size `(seq_len, seq_len, hidden_size // encoder_attention_heads)` output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. )r~r-rrr)rrrrr) rwr~r-rrrresidualrrZoutputss r(rzSpeechT5EncoderLayer.forwards.!)-')+'/ *8* & |Q ]3  =0  6 %(9(9-(HH --m<  "    &Gr*)NNNF) rrrrrkrrrrMrrrs@r(rr ss \~ \ 262604"' ,||,!.,"%,,/ ,   - ,  ,r*rc$eZdZdeffd Z d dej deej deej deej deej deej d eeej d ee d ee fd Z xZ S)SpeechT5DecoderLayerrxct|t|j|j|j d|_tj|j|_ tj|j|j|_ t|j|j|j d|_tj|j|j|_t!||j"|_tj|j|j|_y)NTrr%)rr)rjrkrrdecoder_attention_headsr self_attnrrrrrr'self_attn_layer_norm encoder_attnencoder_attn_layer_normrdecoder_ffn_dimrrr+s r(rkzSpeechT5DecoderLayer.__init__Is *((44,,   zz&"7"78 $&LL1C1CI^I^$_!-     * *,,   (*||F4F4FFLaLa'b$/8N8NO " V-?-?VEZEZ [r*r~r-encoder_hidden_statesencoder_attention_maskrcross_attn_layer_head_maskrr use_cachec |} ||ddnd} |j|| |||\}} } |j|}| |z}|j|}d}d}|R|} ||ddnd}|j||||||\}}}|j|}| |z}|j |}| |z} ||j |z}|j |}|f}|r|| |fz }| r|| fz }|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, hidden_size)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(batch, seq_len, hidden_size)` encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of size `(decoder_attention_heads,)`. past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. Nr)r~rr-rrr)r~rr-rrr)rrrrrrr)rwr~r-rrrrrrrrself_attn_past_key_valueself_attn_weightspresent_key_valuecross_attn_present_key_valuecross_attn_weightscross_attn_past_key_valuers r(rzSpeechT5DecoderLayer.forward_sm<!:H9S>"1#5Y] >Bnn'3)+/ ?M? ; (*; ]3  =0 11-@ (,$! ,$H@N?Yrs(;_c %NRN_N_+!65 :8"3 O`O KM-/K!LL7M$}4M 88GM!24P P &(9(9-(HH --m<  "  )+=> >G  )+ +Gr*)NNNNNNFT) rrrrrkrrrrrMrrrs@r(rrHs\~\2268<9=26=A8<,1$(R||R!.R ( 5 R !) 6 R "%,,/ R%-U\\$:R!u||!45R$D>RD>Rr*rc"eZdZeZdZdZdZdZy)SpeechT5PreTrainedModelspeecht5r+Tc pt|trtjj |j j ddtjd|j jd|j jzz ztjj|j jdyt|trtjd|jjz }tjj!|jj | |tjj!|jj| |yt|tj"rm|j j$j d|j&j(|j%|jj$j+yyt|tj,tj.frJ|jj$j+|j j$j1dyt|tj2rtjj5|j |jjtj|j6|j|jdzz }tjj!|j| |yyt|tj8rz|j j$j d|j&j(|j:2|j j$|j:j+yyy) zInitialize the weightsrrrmeanstd)abr0Nr)rrrinitnormal_rsrrrrg in_channels constant_rir#r) in_featuresr5r(datarxinitializer_rangezero_rrfill_rokaiming_normal_rrr)rwmoduleks r( _init_weightsz%SpeechT5PreTrainedModel._init_weightss f= > GGOO "" !v{{'>'>q'AFKKD[D['["\]]   GG  fkk.. 2  9 : !f//;;;..r*N) rrrr config_classbase_model_prefixmain_input_namesupports_gradient_checkpointingr%r*r(rrs!L"$O&*#?r*rceZdZdZdeffd Z d dejdeejdeejdee dee d ee d e e e ffd ZxZS) SpeechT5Encoderzu Transformer encoder consisting of *config.encoder_layers* layers. Each layer is a [`SpeechT5EncoderLayer`]. rxct||tj|j|j |_tj|j|_ |j|_ tjt|jDcgc] }t|c}|_t#|j|j$z|j&|_d|_|j-ycc}w)Nr%F)rjrkrrrr'rrrrencoder_layerdrop layerdroprrKencoder_layersrrerrencoder_max_relative_positionrr post_initrwrxrZrzs r(rkzSpeechT5Encoder.__init__s  ,,v'9'9v?T?TUzz&"7"78 11mm5QWQfQfKg$ha%9&%A$hi A   &"@"@ @&BfBf ',# %isDr~r- head_maskroutput_hidden_states return_dictr6c ||n|jj}||n|jj}||n|jj}|t ||j }|j |}|j|}|j|}txs t|}|rdnd} |rdnd} |_|jdt|jk7r6tdt|jd|jddt|jD]\} } |r| |fz} d} |j r$t#j$g}||j&k} | r|rY|j(r4|j r(|j+| j,||||| nd||}n| |||||| nd|}|d}| rd }|s| d fz} |r| |fz} |st/d || | fDSt1|| | S) a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): Features extracted from the speech or text input by the encoder prenet. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr*rz&The head_mask should be specified for  layers, but it is for .F)r-rrrr\rc3&K|] }|| ywr|r*.0vs r( z*SpeechT5Encoder.forward..Zsmq_`_lmslast_hidden_stater~ attentions)rxrr5use_return_dictrrBrrrrrrrPrer% enumeraterrrFr/rrr tupler)rwr~r-r4rr5r6r synced_gpusall_hidden_statesall_self_attentionsidx encoder_layerskip_the_layerdropout_probability layer_outputss r(rzSpeechT5Encoder.forwards^H2C1N-TXT_T_TqTq$8$D $++JjJj &1% p 'tn p d^p  uo% &p r*r,ceZdZdZdeffd Z d dejdeejdeejdee dee d ee d e e e ffd ZxZS) SpeechT5EncoderWithSpeechPrenetz Wrapper around SpeechT5Encoder that applies SpeechT5SpeechEncoderPrenet to convert the audio waveform data to hidden features. rxct||t||_t ||_|j yr|)rjrkr/prenetr,wrapped_encoderr2r+s r(rkz(SpeechT5EncoderWithSpeechPrenet.__init__i5  1&9 .v6 r*r+r-r4rr5r6r6c^|j||\}}|j||||||}|SN)r~r-r4rr5r6rRrS rwr+r-r4rr5r6r~rs r(rz'SpeechT5EncoderWithSpeechPrenet.forwardqsG)- L.(Q% ~&&')/!5# ' r*rMrNrs@r(rPrPcs ~26,0,0/3&*''!.ELL)  $D>  'tn d^ uo% &r*rPceZdZdZdeffd ZdZdZ ddejde ejde ejd e e d e e d e e d e eeffd ZxZS)SpeechT5EncoderWithTextPrenetz| Wrapper around SpeechT5Encoder that applies SpeechT5TextEncoderPrenet to convert the input_ids to hidden features. rxct||t||_t ||_|j yr|)rjrkrrRr,rSr2r+s r(rkz&SpeechT5EncoderWithTextPrenet.__init__5  /7 .v6 r*c6|jjSr|rRrr;s r(rz2SpeechT5EncoderWithTextPrenet.get_input_embeddings{{//11r*c:|jj|yr|rRrrs r(rz2SpeechT5EncoderWithTextPrenet.set_input_embeddings ((/r*r+r-r4rr5r6r6cV|j|}|j||||||}|SrVrWrXs r(rz%SpeechT5EncoderWithTextPrenet.forwards@ L1 &&')/!5# ' r*rM)rrrrrrkrrrr^rrrMrrrrrrs@r(rZrZs~20 26,0,0/3&*''!.ELL)  $D>  'tn d^ uo% &r*rZceZdZdZdeffd Z d dejdeejdeejdee dee d ee d e e e ffd ZxZS) SpeechT5EncoderWithoutPrenet This wrapper class is a helper class to correctly load pretrained checkpoints when used in combination with [`SpeechT5Model`]. rxcdt||t||_|j yr|)rjrkr,rSr2r+s r(rkz%SpeechT5EncoderWithoutPrenet.__init__)  .v6 r*r+r-r4rr5r6r6c0|j||||||SrV)rS)rwr+r-r4rr5r6s r(rz$SpeechT5EncoderWithoutPrenet.forwards.##&)/!5# $  r*rMrNrs@r(reres ~26,0,0/3&* '' !. ELL)  $D>  'tn  d^  uo% & r*recVeZdZdZdeffd Z ddeejdeejdeejdeejdeejd eejd ee ejd ee d ee d ee dee de eeffdZxZS)SpeechT5Decoderzt Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`SpeechT5DecoderLayer`] rxct|||j|_t j t |jDcgc] }t|c}|_ d|_ |jycc}wr) rjrkdecoder_layerdropr/rrrKdecoder_layersrrerr2r3s r(rkzSpeechT5Decoder.__init__sd  11mm5QWQfQfKg$ha%9&%A$hi &+#  %isA>r~r-rrr4cross_attn_head_maskrrrr5r6r6c | | n|jj} | | n|jj} ||n|jj}| | n|jj} |j dd} ||ddj dnd} t|| || }||t||j| d}txs t|}|jr%|jr|rtjdd}| rdnd}| rdnd}| r|dnd}|rdnd}t!||gd d gD]j\}}| |j dt#|j$k7s3t'd |d t#|j$d |j ddt)|j$D]\}}| r||fz}d}|jr$t+j,g}||j.k}|r|sE|||nd}|jr?|jr3|j1|j2|||||||nd|||ndd| | }n||||||||nd|||nd|| | }|d}|r ||| rdndfz }| s||dfz}|||dfz}| r||fz}|r|nd}| st5d|||||fDSt7|||||S)a Args: hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, feature_size)`): Features extracted from the speech or text input by the decoder prenet. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr!rr)rzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr*r4rozThe `z` should be specified for r8r9)r-rrrrrrrr rc3$K|]}|| ywr|r*r;s r(r>z*SpeechT5Decoder.forward..s=s)r@rr~rAcross_attentions)rxrr5rrBrr#rrrBrrrrlogger warning_oncerRrPrer%rCrrFr/rr rDr)rwr~r-rrr4rorrrr5r6rrrErFrGall_cross_attentionsnext_decoder_cache attn_mask mask_namerH decoder_layerrJrKrrL next_caches r(rzSpeechT5Decoder.forwardsR2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKK>#A&3t{{+;<$ {*DSEUDVW%NN,Q/03 #,DKK"82 V C#$58H$H!#N}}&+jjn#!4t~~!Ek5D5P_S1VZN**t}} $ A A!**!")*&/&;IcN1E1Q(-W[% ! !.!#1*?+A7@7LYs^RV5I5U,S1[_#1&7' ! *!,M"}:KQQR'S&UU" &9]1=M| $D>| 'tn| d^|  u?? @| r*rkcveZdZdZdeffd Z ddeejdeejdeejdeejdeejd eejd eejd ee ejd ee d ee dee dee de eeffdZxZS)SpeechT5DecoderWithSpeechPrenetz Wrapper around SpeechT5Decoder that applies SpeechT5SpeechDecoderPrenet to convert log-mel filterbanks to hidden features. rxct||t||_t ||_|j yr|)rjrkr`rRrkwrapped_decoderr2r+s r(rkz(SpeechT5DecoderWithSpeechPrenet.__init__rTr*r+r-rrrsr4rorrrr5r6r6c b|j||} |j| ||||||| | | |  }|SN) r~r-rrr4rorrrr5r6rRr)rwr+r-rrrsr4rorrrr5r6decoder_hidden_statesrs r(rz'SpeechT5DecoderWithSpeechPrenet.forwardsS!% L:L M&&/)"7#9!5+/!5#'  r*) NNNNNNNNNNNNr|rs@r(r~r~s1 ~5959=A=A59,07;=A$(,0/3&*u001!!1!12 ((9(9:  !))9)9 :  %U\\2 ELL)'u||4"$u'8'8"9:D>$D>'tnd^ u?? @r*r~cbeZdZdZdeffd ZdZdZ ddee jdee jdee jd ee jd ee jd ee jd ee e jd eedeedeedeedeeeffdZxZS)SpeechT5DecoderWithTextPrenetz{ Wrapper around SpeechT5Decoder that applies SpeechT5TextDecoderPrenet to convert input tokens to hidden features. rxct||t||_t ||_|j yr|)rjrkrrRrkrr2r+s r(rkz&SpeechT5DecoderWithTextPrenet.__init__r\r*c6|jjSr|r^r;s r(rz2SpeechT5DecoderWithTextPrenet.get_input_embeddingsr_r*c:|jj|yr|rars r(rz2SpeechT5DecoderWithTextPrenet.set_input_embeddingsrbr*r+r-rrr4rorrrr5r6r6c j|j|||\} }|j| |||||||| | |  } | Srr)rwr+r-rrr4rorrrr5r6rrs r(rz%SpeechT5DecoderWithTextPrenet.forwardsZ15 L.Zi0j-~&&/)"7#9!5+/!5#'  r*r{)rrrrrrkrrrrr^r]rrrMrrrrrrs@r(rrs%~20 5959=A=A,07;=A$(,0/3&*u001!!1!12 ((9(9:  !))9)9 :  ELL) 'u||4"$u'8'8"9:D>$D>'tnd^ u?? @r*rcVeZdZdZdeffd Z ddeejdeejdeejdeejdeejd eejd ee ejd ee d ee d ee dee de eeffdZxZS)SpeechT5DecoderWithoutPrenetrfrxcdt||t||_|j yr|)rjrkrkrr2r+s r(rkz%SpeechT5DecoderWithoutPrenet.__init__ rhr*r+r-rrr4rorrrr5r6r6c >|j||||||||| | |  } | Sr)r) rwr+r-rrr4rorrrr5r6rs r(rz$SpeechT5DecoderWithoutPrenet.forwardsA&&&)"7#9!5+/!5#'  r*r{r|rs@r(rrs ~5959=A=A,07;=A$(,0/3&*u001!!1!12 ((9(9:  !))9)9 :  ELL) 'u||4"$u'8'8"9:D>$D>'tnd^ u?? @r*rceZdZdZdeffd ZdejdejdejdejfdZ d Z e d Z xZS) $SpeechT5GuidedMultiheadAttentionLossz Guided attention loss from the paper [Efficiently Trainable Text-to-Speech System Based on Deep Convolutional Networks with Guided Attention](https://arxiv.org/abs/1710.08969), adapted for multi-head attention. rxcft||j|_|j|_yr|)rjrkguided_attention_loss_sigmasigmaguided_attention_loss_scalescaler+s r(rkz-SpeechT5GuidedMultiheadAttentionLoss.__init__6s( 77 77 r*rA input_masks output_masksr6cF|j|||j}|jd|jdz}|j|jjd}||z}t j |j |}|j|zS)aY Compute the attention loss. Args: attentions (`torch.FloatTensor` of shape `(batch_size, layers * heads, output_sequence_length, input_sequence_length)`): Batch of multi-head attention weights input_masks (`torch.BoolTensor` of shape `(batch_size, input_sequence_length)`): Input attention mask as booleans. output_masks (`torch.BoolTensor` of shape `(batch_size, output_sequence_length)`): Target attention mask as booleans. Returns: `torch.Tensor` with the loss value r!rr)_make_guided_attention_masksrrrrr masked_selectr)rwrArrguided_attn_masksmaskslosseslosss r(rz,SpeechT5GuidedMultiheadAttentionLoss.forward;s"!==k