UhJddlmZddlmZmZmZmZddlZddlm Z ddl m Z m Z ddl mZddlmZddlmZmZdd lmZdd lmZdd lmZmZmZmZd d lmZd dlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)m*Z*ddl+m,Z,ejZe.Z/eGddeZ0edeGddeZ1Gdde&Z2Gdde'Z3Gdde$Z4Gdde!Z5Gd d!e"Z6eGd"d#e%Z7Gd$d%e jpZ9ed&Gd'd(e#eZ:Gd)d*e jpZ;eGd+d,e%Z<ed-Gd.d/e1e,Z=gd0Z>y)1) dataclass)ListOptionalTupleUnionN)Cache DynamicCache)GenerationMixin)FlashAttentionKwargs)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)Unpack) ModelOutputauto_docstringcan_return_tuplelogging) AutoModel)KwargsForCausalLMLlamaAttentionLlamaDecoderLayerLlamaForCausalLMLlamaMLP LlamaModel LlamaRMSNormLlamaRotaryEmbedding) CsmConfigCsmDepthDecoderConfig)CsmGenerationMixinceZdZUdZdZeejed<dZ ejed<dZ ee e ejed<dZ ee ejdfed<dZ ee ejdfed<dZeejed <dZejed <dZee e ejed <dZee ejdfed <dZee ejdfed <dZeejed<y)CsmOutputWithPastaf Base class for the model autoregressive outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. depth_decoder_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction) of the depth decoder model. depth_decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the depth decoder (scores for each vocabulary token before SoftMax). depth_decoder_past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) depth_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. depth_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. backbone_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction) of the backbone model. Nlosslogitspast_key_values. hidden_states attentionsdepth_decoder_lossdepth_decoder_logitsdepth_decoder_past_key_valuesdepth_decoder_hidden_statesdepth_decoder_attentions backbone_loss)__name__ __module__ __qualname____doc__r%rtorch FloatTensor__annotations__r&r'rr(r)r*r+r,r-r.r/u/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/csm/modular_csm.pyr$r$/s(*X)-D(5$$ %, $FE  $AEOXeE%*;*;$<=>E=AM8E%"3"3S"89:A:>Ju00#567>6:!2!23:.2%++2OS!8E%8I8I2J,K#LSKO%0A0A30F*G!HOHLhuU->->-C'DEL15M8E--.5r8r$z[ The bare Csm Model outputting raw hidden-states without any specific head on top. ) custom_introcBeZdZeZdZdZdgZdgZdZ dZ dZ dZ dZ dZdZy)CsmPreTrainedModelmodelTCsmDecoderLayerr'c6|jj}t|tjrY|j j jd||j%|jj jyyt|tjrf|j j jd||j2|j j |jjyyt|trJ|j}t|dz D],}|j j |jd|.yt|tr&|j j j!dyy)Ng)meanstdrg?)configinitializer_range isinstancennLinearweightdatanormal_biaszero_ Embedding padding_idxCsmCodebooksHead num_codebooksrange CsmRMSNormfill_)selfmodulerArOis r9 _init_weightsz CsmPreTrainedModel._init_weightss8kk++ fbii ( MM   & &CS & 9{{&   &&('  - MM   & &CS & 9!!- ""6#5#56<<>.  0 1"00M=1,- A ""1%--3C-@ A  + MM   $ $S ),r8N)r0r1r2r config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_cache_class_supports_quantized_cache_supports_static_cache_supports_attention_backendrVr7r8r9r<r<jsQL&*#*+#4"5!N! $!"&*r8r<c eZdZy)rQNr0r1r2r7r8r9rQrQr8rQc eZdZy)CsmRotaryEmbeddingNrcr7r8r9rfrfrdr8rfc eZdZy)CsmMLPNrcr7r8r9rhrhrdr8rhc eZdZy) CsmAttentionNrcr7r8r9rjrjrdr8rjc eZdZy)r>Nrcr7r8r9r>r>rdr8r>cBeZdZeZfdZee ddejde ejde ejde ejde e de ejde ed e ed e ed e ejd eed eeeffdZxZS)CsmDepthDecoderModelct||tj|j|j z|j |_tj|j |jd|_ yNF)rJ) super__init__rErLrO vocab_sizebackbone_hidden_size embed_tokensrF hidden_sizeinputs_embeds_projectorrSrB __class__s r9rqzCsmDepthDecoderModel.__init__s]  LL&*>*>ARAR*RU[UpUpq')yy1L1LfN`N`gl'm$r8 input_idsbackbone_last_hidden_stateattention_mask position_idsr' inputs_embeds use_cacheoutput_attentionsoutput_hidden_statescache_positionflash_attn_kwargsreturnc B|5tjjstj dd}||n|j j }| | n|j j} ||n|j j}|du|duz r td|jr%|jr|rtj dd}|r | t}| i||jnd} ||jdn|jd} | |jn |j}tj | | | z|} |tj"| dz d }||j$z}|j'||z}| ddk(}| ||dddf<n5tjjs|rtj)d |j+|}|j-||| ||}|}| j/d}|j1||}| rd nd}|rd nd}|j2d|j j4D],}| r||fz }||f|||||| |d | }|d}|s$||dfz }.|j7|}| r||fz }t9||r|nd|| S)aJ backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*): The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model) is provided in the `input_ids` argument. NzCustom `position_ids` were provided but will be ignored. CSM depth decoder automatically determines position_ids from `cache_position` and as it requires them to be identical across the batch, the provided position_ids will be ignored.z;You must specify exactly one of input_ids or inputs_embeds.zX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Frrdevice)minzvWhen the first codebook token is provided, `backbone_last_hidden_state` should also be provided for correct inference.r7)r{r|past_key_valuerr~rposition_embeddings)last_hidden_stater'r(r))r4compiler is_compilinglogger warning_oncerBrrr~ ValueErrorgradient_checkpointingtrainingr get_seq_lengthshaperarangeclamprrrtwarningrv_update_causal_mask unsqueeze rotary_emblayersnum_hidden_layersnormr )rSryrzr{r|r'r}r~rrrrpast_seen_tokensinputs_seq_lengthr codebook_idxsoffsetinput_ids_are_first_codebook causal_maskr(rall_hidden_statesall_self_attns decoder_layer layer_outputss r9forwardzCsmDepthDecoderModel.forwards*  #ENN,G,G,I   M  L1B1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKK?L] & &//2 "oom\J#7BD0d![[)H4;;+H+HI 6M#!m%55!) *)."3#-$7 $ M*!,M =#3"55' 6* -0  -!1 1 &+/8Od+%   r8) NNNNNNNNNN)r0r1r2r!rWrqrrr4 LongTensorrr5Tensorr boolrr rrr r __classcell__rxs@r9rmrms*(Ln '+BF1537+/59$(,0/359p ##p %-U->->$?p !. p u//0 p "% p   1 12p D>p $D>p 'tnp !!1!12p $$89p  u-- .p p r8rmc&eZdZfdZddZxZS)rNct|||_tjt j |jdz |||_y)Nr)rprqrOrE Parameterr4emptyrG)rSrurOrrrxs r9rqzCsmCodebooksHead.__init__%s? *ll5;;t/A/AA/E{T^#_` r8c |2|jd}|jtj|}n|dz }|j|}t |jdDcgc]9}t j j|dd|ddf||j;}}tj|d}|Scc}w)Nrrdim) rrGr4rrPrE functionallinearTstack)rSr(r seq_lengthcodebook_weightr codebook_idxs r9rzCsmCodebooksHead.forward*s  !&,,Q/J"kk%,,z*BCO*Q.M"kk-8O!&o&;&;A&> ?  MM q,/A!BOT`DaDcDc d   Mq9   s#>B<Nr0r1r2rqrrrs@r9rNrN$sa r8rNa$ The CsmDepthDecoder Model transformer, with a [`CsmCodebooksHead`] on top, which can be seen a position-specific language modeling head, allowing to use a different linear layer for each codebook (e.g. position 0 is the first codebook and uses the first codebook head, etc.) c!PeZdZdZdZdZfdZdZdZ dde jde e de e jde e jd e e jf fd Zee dde jd e e jde e j"d e e jde ee ee jfde e jd e e jde ede ede ed e e jdeee j"fdeedeeeffdZxZS)CsmDepthDecoderForCausalLMNct|||`t|j|j |j |_t||_ yr) rprqlm_headrNrurOrrcodebooks_headrmr=rws r9rqz#CsmDepthDecoderForCausalLM.__init__GsE  L.v/A/A6CWCWY_YjYjk)&1 r8ctdNzNot needed for CsmAttributeErrorrSs r9get_output_embeddingsz0CsmDepthDecoderForCausalLM.get_output_embeddingsM122r8ctdrrrSnew_embeddingss r9set_output_embeddingsz0CsmDepthDecoderForCausalLM.set_output_embeddingsPrr8ryr'r{r}rc t ||||||fi|}|dddk(}|s|jd|jd|S)Nrrrzr|)rpprepare_inputs_for_generationpop) rSryr'r{r}rkwargs model_inputsis_first_generation_steprxs r9rz8CsmDepthDecoderForCausalLM.prepare_inputs_for_generationSsgw<  ~ Y_ $00@#A!#D#I '   9 : (r8rzr|labelsr~rrlogits_to_keeprrc  `| | n|jj} | | n|jj} |jd|||||||| | | d | }|d}t | t r | dk(r t dd}nt | d}n| }|j|dd|ddf| | |nd}|j}d}|B|dddfj}|jd|d|jj|d| }t|||j|j|jS) a backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*): The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model) is provided in the `input_ids` argument. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. N) ryrzr{r|r'r}r~rrrrr.)r&rrr shift_labels)r%r&r'r(r)r7)rBrrr=rDintslicer contiguous loss_functionrrrr'r(r))rSryrzr{r|r'r}rr~rrrrroutputsr( slice_indicesr&r%rs r9rz"CsmDepthDecoderForCausalLM.forwardisy62C1N-TXT_T_TqTq$8$D $++JjJj  $**  'A)%+'/!5)      nc *" %a %~ot < *M$$ !]A- .Q_Qk}0Mqu ""$  !#qr'?557L%4%%dt{{7M7M\hlrD&#33!//))   r8NNNN) NNNNNNNNNNNr)r0r1r2_tied_weights_keys_tp_plan_pp_planrqrrr4rrr r5rrrrrrrrrrrrrrrs@r9rr;sHH2 33 ,0595959 ##"%!!1!12    1 12  !!1!12 ,'+BF1537KO59-1$(,0/35934J ##J %-U->->$?J !. J u//0 J "%tE4E4E/F(F"GH J   1 12J ))*J D>J $D>J 'tnJ !!1!12J c5<</0J *+J  u,, -J J r8rc$eZdZfdZdZxZS)CsmBackboneModelEmbeddingsct|tj|j|j z|j |_|jdtj|j|j zdy)Naudio_tokens_offsetsF) persistent) rprqrErLrOrrruembed_audio_tokensregister_bufferr4rrws r9rqz#CsmBackboneModelEmbeddings.__init__sn "$,,0D0DvGXGX0X[a[m[m"n  "ELL1E1E$FIZIZ$Zgl  r8cf|j||jz}|jd}|S)Nrr)rrsum)rSry input_embedss r9rz"CsmBackboneModelEmbeddings.forwards6..y4;T;T/TU #''A'. r8rrs@r9rrs  r8rc<eZdZfdZeefdZxZS)CsmBackboneModelcDt||t||_yr)rprqrrtrws r9rqzCsmBackboneModel.__init__s  6v>r8c "t|di|S)a& input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`): 1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens. 2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) r7)rpr)rS super_kwargsrxs r9rzCsmBackboneModel.forwardsw...r8)r0r1r2rqrrrrrs@r9rrs$? / /r8rz The Csm model consists of two llama-like auto-regressive transformer models: a backbone model that predicts the first codebook token and a depth decoder that predicts the other codebook tokens. c#eZdZddgZfdZdZdZdZdZdZ e fd Z fd Z d"d e ejd e ejd e ejde ejde ejf dZ d"d ej"de ede ej"de ej&de ej"f fd Zee d#d ej"d e ejde ejd e ejde ej"de eeeej&fde ej&de ej"de ede ede ede ej"deeejfdeedeeeffdZe dejdededejBdejd ef d!Z"xZ#S)$CsmForConditionalGenerationz5backbone_model.embed_tokens.embed_audio_tokens.weightz'depth_decoder.model.embed_tokens.weightct|||j|_tj|j |jd|_tj|j|j |_ tj||_ tj|j|_t!j"|j$|_|j)yro)rprqrrrErFrurrLtext_vocab_sizeembed_text_tokensr _from_configbackbone_modelrdepth_decoder_config depth_decoderr from_config codec_config codec_model post_initrws r9rqz$CsmForConditionalGeneration.__init__s   ++yy!3!3V5F5FUS !#f.D.DfFXFX!Y.;;FC7DDVE`E`a$001D1DE r8c.|jjSrrrtrs r9get_input_embeddingsz0CsmForConditionalGeneration.get_input_embeddingss""///r8c&||j_yrr)rSvalues r9set_input_embeddingsz0CsmForConditionalGeneration.set_input_embeddingss+0(r8c|jSrrrs r9rz1CsmForConditionalGeneration.get_output_embeddingss ||r8c||_yrrrs r9rz1CsmForConditionalGeneration.set_output_embeddingss % r8c|jjrO|j|jjj |j jjyyr)rBtie_codebooks_embeddings_tie_or_clone_weightsrrtrrr=rs r9 _tie_weightsz(CsmForConditionalGeneration._tie_weightssL ;; / /  & &##00CC""((55  0r8c|jddrt ||i|\}}nt ||i|}d}t|}t |j j Dcic]\}}|j|r||d|} }}t |jj jddi| | D]}t|j ||zd|vr|fS|Scc}}w)Noutput_loading_infoFdepth_decoder__from_model_config) getrpfrom_pretrainedlenvarsgeneration_configitems startswithrupdatedelattr) clsargsrr= loading_infoprefix prefix_lenattrrdepth_decoder_attrsrxs r9rz+CsmForConditionalGeneration.from_pretraineds ::+U 3"''"94"J6"J E<G+Tk(}|jA|jCd||<|e|jdjAdd|j,j6}||||<|d k(jEd }d ||d|dddf<|}||dScc}w)a Merges the input_ids and input_values to produce a single inputs_embeds tensor: 1 - Infers the codec model on the input_values to retreive codebook token. 2 - Embeds codebook tokens and places them at the correct positions in the inputs_embeds tensor. 3 - If labels are provided, expands them to match codebook dimensions and position the target codebook tokens in the inputs_embeds tensor. Args: input_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`): The input ids to embed. input_values (`torch.Tensor` of shape `(batch_size, channels, audio_sequence_length)`): The audio input values to embed. input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`): The cutoffs of the audio input values relative to its batch index, padded with -1 when no audio. Nrrrrr.c3:K|]}|jdyw)rN)r).0els r9 zQCsmForConditionalGeneration._merge_input_ids_with_input_values..\s"K2288A;"Ks)rdtypeiTas_tuple)r}r)#rrErpaddiffr4rmaxrexpandrrziprPrrencode audio_codes transposeappendrget_audio_codes_maskrBaudio_token_idrrtonesrOlongcodebook_eos_token_idsqueezeaudio_eos_token_idrepeatrnonzero)rSryr#r$rr} audio_lengthsinput_values_maskaudio_tokens_listbatch_input_valuesbatch_input_values_cutoffsrU start_idxend_idx audio_batch codec_outputs codebook_idsmax_audio_framesr*batched_audio_token_idsaudio_codes_maskr:audio_token_mask audio_embedsaudio_eos_frame_idsaudio_eos_embedsaudio_eos_token_masklabels_expanded depth_decoder_ignore_frames_idxss r9"_merge_input_ids_with_input_valuesz>CsmForConditionalGeneration._merge_input_ids_with_input_values-sl*..y9  ##%==#4#45I6#R 01E1JKPPRM)-!*;>"$>-GHbfgHg-h*9??BQFG>A :1 =I8Q?G"4S)G:K5K"LK$($4$4$;$;K > #"K9J"KK &+kk\mnVX""21a1ABHHQK1O'PQn' # $//DDEVW ![[77N(N: ..;;>> import torch >>> from transformers import CsmForConditionalGeneration, AutoProcessor >>> from datasets import load_dataset, Audio >>> model_id = "eustlb/csm-1b" >>> torch_device = "cuda" if torch.cuda.is_available() else "cpu" >>> processor = AutoProcessor.from_pretrained(model_id) >>> ds = load_dataset("hf-internal-testing/dailytalk-dummy", split="train") >>> # ensure the audio is 24kHz >>> ds = ds.cast_column("audio", Audio(sampling_rate=24000)) >>> conversation = [] >>> # prepare a conversation with text and corresponding audio >>> for text, audio, speaker_id in zip(ds[:4]["text"], ds[:4]["audio"], ds[:4]["speaker_id"]): ... conversation.append( ... { ... "role": f"{speaker_id}", ... "content": [{"type": "text", "text": text}, {"type": "audio", "path": audio["array"]}], ... } ... ) >>> inputs = processor.apply_chat_template( ... conversation, ... tokenize=True, ... return_dict=True, ... output_labels=True, ... ).to(torch_device) >>> model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=torch_device) >>> output = model(**inputs) >>> output.loss.backward() ```Nrr}r) ryr{r|r'r}r~rrrr)r&rrrrr/r'r.r&)rTr-)ryrzr~rr return_dictr) r%r/r*r&r'r(r)r+r,r-r.r7)rBrrrXrVrrDrrrrrrallrOrErr0rArr%r$r'r(r)r&)rSryr#r{r$r|r'r}rr~rrrrrrYbackbone_outputsbackbone_hidden_statesrbackbone_logitsr%r/r*depth_decoder_outputsbackbone_labels train_maskdepth_decoder_input_ids train_idxsbackbone_last_hidden_statesdepth_decoder_labelss r9rz#CsmForConditionalGeneration.forwardsl2C1N-TXT_T_TqTq$8$D $++JjJj   Y^^q%8 CC<)=vM*/:M"8,FI.4..  )%+'/!5)    "2!!48B>SV8W~ot4]k ,,'=aPQ>Q'RS ! $  $Q1WoO.D..&4;;KaKaekM "!Q(+t388R8@@J&,Z&8>] @Y@Y\]@]>]9]&^ #&(mm&7&78OQW_`&7&a ##++T+:J*@APZ[\P]`aPacdAd*e '#)*#5 $($6$61+F#"3%9 +%7% !"7!;!;  #55D '1",<<*88'22AVAb!6!=!=hl$0+@*O*O$0)>(K(KI^Ij%:%E%E  qu  r8sequence_length target_lengthr, batch_sizec ||jdk(r|}|Stj|j}tj||f|||j }|dk7rtj |d}|tj||j |jddkDz}|ddddddfj|ddd}||j}|jd} |ddddddd| f|ddddddfj|j z} | dk(} |ddddddd| fj| ||ddddddd| f<|S) aM Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. N) fill_valuer,rr)diagonalrr'r)rr4finforfullrtriurreshaper3clonerto masked_fill) r{rgrhr,rrirr min_dtype mask_length padding_masks r95_prepare_4d_causal_attention_mask_with_cache_positionzQCsmForConditionalGeneration._prepare_4d_causal_attention_mask_with_cache_positionFs<  %.*<*<*>!*C(K*' E*..I** -0Ye\j\q\qK!##jjqA 5<< n>S>STWeWmWmnprsWtt tK%dD!Q&67>>z1bRTUK))//1 ,2226 *1aL[L+@ANSTVZ\`bcScDdDgDg&&E   ,q0 5@Aq,;,AV5W5c5c )6 Aq!\k\12r8r) NNNNNNNNNNNNr)$r0r1r2rrqrrrrr  classmethodrr"rr4rrVrr r5rrrrrrrrrrr$r staticmethodr,rxrrs@r9rrs  @1 01&41-1/37;)- NBELL)NBu||,NB'u||4 NB & NB %,,  NBf,0595959 ##"%!!1!12    1 12  !!1!12 >'+/3157;37KO59-1$(,0/35934f ##f u||,f !. f 'u||4 f u//0 f "%tE4E4E/F(F"GHf   1 12f ))*f D>f $D>f 'tnf !!1!12f c5<</0f *+f u'' (!f f P4 444{{ 4  4  44r8r)r<rrmrr)? dataclassesrtypingrrrrr4torch.nnrE cache_utilsr r generationr modeling_flash_attention_utilsr modeling_outputsr rmodeling_utilsrprocessing_utilsrutilsrrrrautorllama.modeling_llamarrrrrrrrconfiguration_csmr r!generation_csmr" get_loggerr0rr$r<rQrfrhrjr>rmModulerNrrrr__all__r7r8r9rs "// .)BO-&KK   @.   H % 76 76 76t **  *D   -  X  >  ' z :z z zryy.s !1?s s l  /z//. X"46HX  Xv  r8