Uh+RdZddlZddlmZddlmZddlmZmZm Z m Z ddl Z ddl Z ddl m Z ddlmZdd lmZdd lmZmZmZmZmZmZd d lmZej6eZdad ZGdde j@jBZ"d%dZ#d%dZ$Gdde jJZ&Gdde jJZ'Gdde jJZ(eGddeZ)eGddeZ*eGddeZ+eGdde)Z,ed !Gd"d#e)eZ-gd$Z.y)&zPyTorch RWKV model.N) dataclass)Path)ListOptionalTupleUnion)nn)GenerationMixin)PreTrainedModel) ModelOutputauto_docstringis_bitsandbytes_availableis_ninja_availableis_torch_cuda_availablelogging) RwkvConfigcddlm}ttj j j j dz dz }dDcgc]}||z  }}t t j|k(rytjd|ddd d d d d d|g}|d||tjtjk(|a|t _ycc}w)Nr)loadkernelsrwkv)z wkv_op.cppz wkv_cuda.cuzwkv_cuda_bf16.cuz2Loading CUDA kernel for RWKV at context length of .z -res-usagez--maxrregcount 60z--use_fast_mathz-O3z -Xptxas -O3z--extra-device-vectorizationz-DTmax=wkv_)namesourcesverboseextra_cuda_cflags) torch.utils.cpp_extensionrr__file__resolveparentrwkv_cuda_kernelmax_seq_lengthloggerinfor get_verbosityDEBUG)context_length load_kernel kernel_folderfcuda_kernel_filesflagss x/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/rwkv/modeling_rwkv.pyload_wkv_cuda_kernelr0.s=N**,33::AAIMPVVM4efq*ff#(8(G(G>(Y KKD^DTTUVW  & .!" E#N# $!&&(GMM9  '5#/gs Cc0eZdZeddZeddZy)RwkvLinearAttentionNc |j\}}} |tjkDr td|dtjd|| zt | dzdk7rtd|d| dt | dd |j |_|jjd k7sK|jjd k7s2|jjd k7s|jjd k7r td tj|jj }|j tjk(r0|j}|j}|j}|j}|j}|j}tj|tj } |s||Vtj || d tj"|jtj}|dddddfxxdzcc<nBtj$|D cgc]} | j'dc} dj}|j tj(k(rtj*} ntj,} | ||||| |nI|j tj(k(rtj.ntj0} | ||||| |j3||||| |4tj4|d dD cgc]} | j7d}} | j9|j |fScc} wcc} w)NzCannot process a batch with z+ tokens at the same time, use a maximum of z with this model. rzThe product of batch size (z) and hidden size (z") needs to be a round multiple of rcudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices. memory_formatr )dtypedevicer7籡*G)dim)sizer#r$ ValueErrorminr8 input_dtyper9typetorchexpfloat contiguousfloat16 empty_likecontiguous_formatzerosfloat32cat unsqueezebfloat16forward_with_state_bf16forward_with_state forward_bf16forwardsave_for_backwardchunksqueezeto) ctx time_decay time_firstkeyvaluestate return_state batch_sizeseq_len hidden_sizeoutputs forward_funcs r/rQzRwkvLinearAttention.forwardOs+.88:( G[ %44 4.wi7b#2233DF   #c+r&: :a ?-j\9L[MZ";34A7  ))    " "f ,  %%/zz&(||  F*tu uii 0 0 2 = = ?@@ 99 %#))+J))+CKKME**, nn  "!!#U5L5LM 5,} --::"'"9"9 aAg$& 5"Aa1;;q>"AqITTVyyENN*/GG /BB ZeVU K>$A%..u}} '' %BYBYZ   E4K4KL""58O8OP %-- '~~'H:E:W(66]m]v]v          !      OOK ( OOK ( HH[ ! JJ{ #     NFN)__name__ __module__ __qualname__ staticmethodrQrfrnr/r2r2Ns)<1<1|% % rnr2c|j\}}}tj|}|tj|dddftj} tj|dddftj} tj|dddftjdz } n|\} } } tj| }t |D]} |dd| fj } |dd| f}tj| | |z}tj| |z }tj| |z|z }|| z||zz}|| z|z}||z j|j|dd| f<tj| |z| }tj| |z|z }tj| |z }|| z||zz} || z|z} |}  |s|| | | g}||fS)Nr)r8r;) r=rB zeros_likerJrCrangerDmaximumrUr8)rWrXrYrZr[r\_ seq_lengthr` num_state den_state max_state current_index current_key current_valuemax_for_outpute1e2 numerator denominator max_for_states r/rwkv_linear_attention_cpursxxzAz1   c "F }$$SAYemmD $$SAYemmD $$SAYemmDtK */' 9i ))J''Jz*" !]*+113 a./ y+ 2JK YYy>1 2 YY{Z/.@ ANR-%77 9nr) $- $;#?#? #Mq-  i*& YY{]2 3NR-%77 NR' ! %"(u(Iy1 5=rnctd||||fD}|jddk(}t|s|rt||||||Stj ||||||S)Nc3NK|]}|jjdk7yw)r5N)r9rA).0ts r/ z(rwkv_linear_attention..sXa!((--6)Xs#%rr[r\)anyr=r#rr2apply)rWrXrYrZr[r\no_cuda one_tokens r/rwkv_linear_attentionrsmXJ CQV3WXXG q I7i(ZeSXgstt"((ZeUT`aarnc0eZdZdfd ZddZddZxZS)RwkvSelfAttentioncrt|||_tduxrtj|j k(}t r"tr|s t|j ||_ |j}|j |jn|}||_tj t#j$||_tj t#j$||_tj t#j$dd||_tj t#j$dd||_tj t#j$dd||_tj0d|_tj4||d|_tj4||d|_tj4||d|_tj4||d|_y#t$rtjdYwxYw)Nz9Could not load the custom CUDA kernel for RWKV attention.rrrrFbias)super__init__configr#r$r)rrr0 Exceptionr%r&layer_idr_attention_hidden_sizer ParameterrBemptyrWrX time_mix_keytime_mix_valuetime_mix_receptance ZeroPad2d time_shiftLinearrYrZ receptancer`)selfrr kernel_loadedr_r __class__s r/rzRwkvSelfAttention.__init__s  (4q9I9X9X\b\q\q9q  $;$=m Y$V%:%:;! (( ,2,H,H,TF ( (Ze &;",,u{{3H'IJ,,u{{3H'IJLLQ;)GH ll5;;q![+IJ#%<< Aq+0N#O ,,}599[*?eLYY{,AN ))K1FUSii 5{O ) Y WX YsHH65H6cp|jddk(r||ddddd|jf}n3|j|}| |ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}||j z|d|j z zz}|j |}|j|}tj|j|}| |dddf|ddddd|jf<||||fSNrrr) r=rrrrrrYrZrBsigmoidr)rhiddenr[shiftedrYrZrs r/extract_key_valuez#RwkvSelfAttention.extract_key_values< ;;q>Q 5#4Ahq!T]]23Goof-G %aAt}})< =1 t(((7a$:K:K6K+LL,,,w!d>Q>Q:Q/RRd666AH`H`D`9aa hhsm 5!]]4??:#>?  ,21b5ME!HQ4==( )3u,,rncj||\}}}}|tfd|ddDnd}tjj||||\}}|T|d|dddddj f<|d|dddddj f<|d|dddddj f<j ||z|fS) Nr[c3JK|]}|ddddjfywrpr)rrars r/rz,RwkvSelfAttention.forward..#s!FqAaDMM12Fs #r:rrrr )rtuplerrWrXrr`) rrr[ use_cacherrYrZ layer_staters ` r/rQzRwkvSelfAttention.forward!s(,(>(>vU(>(S% CJOJ[eFE!"IFFae 1 OO OO  "  k  ",7NE!HQ4==( ),7NE!HQ4==( ),7NE!HQ4==( ){{:,-u44rnrrpro)rqrrrsrrrQ __classcell__rs@r/rrsP<-&5rnrc(eZdZdfd ZddZxZS)RwkvFeedForwardcBt|||_||_|j}|j |j nd|jz}t jd|_t jtjdd||_ t jtjdd||_ t j||d|_t j||d|_t j||d|_y)NrrrFr)rrrrr_intermediate_sizer rrrrBrrrrrYrrZ)rrrr_rrs r/rzRwkvFeedForward.__init__6s    (( (.(@(@(LF $ $RSV\VhVhRh ,,}5LLQ;)GH#%<< Aq+0N#O 99[*;%H))K5IYY0+EJ rncz|jddk(r||ddddd|jf}n3|j|}| |ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}t j t j|j|}|j|}t j|j|}| |dddf|ddddd|jf<||z|fSr) r=rrrrrBsquarerelurYrZrr)rrr[rrYrrZs r/rQzRwkvFeedForward.forwardGs) ;;q>Q 5#4Ahq!T]]23Goof-G %aAt}})< =1 t(((7a$:K:K6K+LLd666AH`H`D`9aa ll5::dhhsm45 3]]4??:#>?  ,21b5ME!HQ4==( )E!5((rnrrprqrrrsrrQrrs@r/rr5sK")rnrc&eZdZfdZddZxZS) RwkvBlockct|||_||_|dk(r0t j |j |j|_t j |j |j|_ t j |j |j|_ t|||_ t|||_y)Nr)eps)rrrrr LayerNormr_layer_norm_epsilonpre_lnln1ln2r attentionr feed_forward)rrrrs r/rzRwkvBlock.__init__\s    q=,,v'9'9v?X?XYDK<< 2 28Q8QR<< 2 28Q8QR*68<+FH=rnc|jdk(r|j|}|j|j|||\}}||z}|j |j ||\}}||z}||f}|r||fz }|S|dz }|S)Nr)r[rrrp)rrrrrr)rrr[routput_attentionsrroutputss r/rQzRwkvBlock.forwardjs ==A [[(F>>$((6*:%S\>] 5)#"//0@/N e,&5/   | #G w Grn)NFFrrs@r/rr[s  >rnrc0eZdZeZdZdgZddgZdZdZ dZ y)RwkvPreTrainedModelrrrWrXTc t|trv|j}|jj}|jj }|j }||dz z }d||z z }tjt|Dcgc]}||z  c}|jj|jj} | ddddf} t|D cgc]} dd| |dz z dd|zzzzz} } tj| |jj|jj} tjt|Dcgc] }|dzd zdz c}|jj|jjd z} tj5| |j_tj"|jt%j&d z| z|j_tj(| ||j_tj(| |d |zz|j*_tj(| d |z|j,_dddyt|t.r |j}|jj}|jj }d||z z }tjt|Dcgc]}||z  c}|jj|jj} | ddddf} tj5tj(| ||j_tj(| ||j,_dddyycc}wcc} wcc}w#1swYyxYwcc}w#1swYyxYw) zInitialize the weights.rg?r8r9Ngffffff?g?r g?g333333?) isinstancerrrnum_hidden_layersr_rrBtensorrxrr8r9rWrXno_graddata ones_likemathlogpowrrr) rmodulerrr_r ratio_0_to_1ratio_1_to_almost0i time_weighth decay_speedzigzags r/ _init_weightsz!RwkvPreTrainedModel._init_weightss] f/ 0H & ? ?  --33K$*$@$@ !#'81'<=L!$3D(D!E ,,*/ *<=Q[=))//**11K &dD!m4K45Q!4q89sS L c c>  ] ]s2 N&N+N0.CN55 OA O5N>ON) rqrrrsr config_classbase_model_prefix_no_split_modules_keep_in_fp32_modulessupports_gradient_checkpointing _is_statefulrrurnr/rr}s1L$ )<8&*#L7]rnrceZdZUdZdZeejed<dZ ee ejed<dZ ee ejdfed<dZ ee ejdfed<y) RwkvOutputa Class for the RWKV model outputs. Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nlast_hidden_stater[. hidden_states attentions)rqrrrs__doc__rrrB FloatTensor__annotations__r[rrrrrurnr/rrsw,6:x 1 129/3E8D**+ ,3=AM8E%"3"3S"89:A:>Ju00#567>rnrceZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejdfed<dZee ejdfed<y) RwkvCausalLMOutputa| Base class for causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nlosslogitsr[.rr)rqrrrsrrrrBrrrr[rrrrrurnr/rrs0)-D(5$$ %,*.FHU&& './3E8D**+ ,3=AM8E%"3"3S"89:A:>Ju00#567>rnrceZdZfdZdZdZe ddeejdeejdeejdee ejdee d ee d ee d ee d e eeffd ZdZdZxZS) RwkvModelc t||tj|j|j |_tjt|jDcgc]}t||c}|_ tj|j |_ d|_d|_|j!ycc}w)NrF)rrr Embedding vocab_sizer_ embeddings ModuleListrxrrblocksrln_outlayers_are_rescaledgradient_checkpointing post_init)rridxrs r/rzRwkvModel.__init__s  ,,v'8'8&:L:LMmmPUV\VnVnPo$pYv%D$pq ll6#5#56 #( &+# %qs&C c|jSrprrs r/get_input_embeddingszRwkvModel.get_input_embeddingss rnc||_yrpr rnew_embeddingss r/set_input_embeddingszRwkvModel.set_input_embeddingss (rn input_idsattention_mask inputs_embedsr[rroutput_hidden_states return_dictreturnc ||n|jj}||n|jj}||n#|js|jjnd}||n|jj }|t jd|j|jk(r|j| | td| | td||j|}|r||jd|jj|jjf} tdD cgc]A} t!j"| | dkr |j$nt j&|j(d C}} |d xxd zcc<|j*r%|jr|rt jd d}|} |rd nd} |rd nd} t-|j.D]\}}|j*r0|jr$|j1|j2| |||\} }}n|| |||\} }}|jr=|jj4dkDr$|dz|jj4zdk(r| dz } |r| | fz} |s| |fz} |j7| } |r| | fz} |st9d| || | fDSt;| || | Scc} w)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrrrrgꌠ9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...ru)r[rrr:c3&K|] }|| ywrpru)rxs r/rz$RwkvModel.forward..{stqfgfsts)rr[rr)rrrtrainingruse_return_dictr% warning_oncer_rescale_layersr>rr=r_rrxrBrIr8rJr9r enumerater_gradient_checkpointing_func__call__ rescale_everyrrr)rrrrr[rrrrshaperrall_self_attentionsall_hidden_statesr blockrs r/rQzRwkvModel.forwards>2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IZ^ZgZgT[[=R=Rmr %0%cd d  =#8TU U   OOI6M "''*DKK,C,CT[[EbEbcE q  a-"5"5U]][h[o[oE !H H  & &4==##p" % $5b4"6BD#DKK0 JJC**t}}373T3TNNM5)EV40 uj49!)Wh40 uj ((KK--11W 9 99Q> - 1 #$58H$H! &9ZM&I#+ J. M2  1]4D D t]E;LNa$btt t++*   es5AK c |j|j k(ry|jjdkDrt j 5t |jD]\}}|jr|jjjjdt||jjzz|jjjjdt||jjzzt|jjjdr|jjjj j#dt||jjzz|jjjj j#dt||jjzzt|jjjdrN|j%|jj||j%|jj||jjjj#dt||jjzz|jjjj#dt||jjzz ddd|j |_y#1swYxYw)Nrr:SCB quant_state)rrrr$rBrr!rrr`weightmul_intrrZhasattrr*div_ _bnb_4bit_dequantize_and_rescale)rblock_idr(s r/r zRwkvModel._rescale_layerss`  # #DMM(9 :  ;; $ $q ( r'0'=rOHe}}..55::1HPTP[P[PiPiDi@j;jk**0077<>qCTXT_T_TmTmHmDn?no!..44;;@@c(VZVaVaVoVoJoFpApqr r"(,}}#4 # r rs KL77Mcts tdddl}|jj |j j |j j}|jdt||jjzz|jj|jddj|j}t!|d|y) z Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will be quantized again. z/Please install bitsandbytes to use this method.rNr:cpuF) requires_gradr,)r ImportError bitsandbytes functionaldequantize_4bitr,rr+r0r.rr$r Params4bitrUr9setattr)r target_layerr2bnbdequant_weights quant_weights r/r1z*RwkvModel._bnb_4bit_dequantize_and_rescales )*OP P"..889L9L9Q9QS_SfSfSrSrsQ#h$++2K2K&K"LLMvv((););E)BRW(X[[\k\r\rs  h 5rn)NNNNNNNN)rqrrrsrrrrrrB LongTensorrrboolrrrrQr r1rrs@r/rrs )15595937$(,0/3&*l E,,-l !!1!12l   1 12 l U../0 l D> l $D>l 'tnl d^l  uj !l l \506rnrz The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introc0eZdZdgZfdZdZdZddZe dde e jde e jde e jd e e e jd e e jd e ed e ed e ede edeeeffdZxZS)RwkvForCausalLMz head.weightct||t||_t j |j |jd|_|jy)NFr) rrrrr rr_rheadr )rrrs r/rzRwkvForCausalLM.__init__sH  f% IIf00&2C2C%P  rnc|jSrprFr s r/get_output_embeddingsz%RwkvForCausalLM.get_output_embeddingss yyrnc||_yrprHrs r/set_output_embeddingsz%RwkvForCausalLM.set_output_embeddingss " rnc h||dddfjd}||d|i}nd|i}||d<||d<|S)Nrrrr[r)rL)rrr[rrkwargs model_inputss r/prepare_inputs_for_generationz-RwkvForCausalLM.prepare_inputs_for_generations]  !!R%(2226I  $+];L'3L % W$- [!rnrrrr[labelsrrrrrc t| | n|jj} |j||||||| } | d} |j| } d}|*|j| |fd|jj i| }| s| f| ddz}||f|zS|St || | j| j| jS)aI input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. N)rr[rrrrrrr)rrr[rr) rrrrF loss_functionrrr[rr)rrrrr[rPrrrrrM rwkv_outputsrrrr`s r/rQzRwkvForCausalLM.forwardsJ&1%F $D>F 'tnF d^F  u(( )F F rnrD)rDrrro)/rr dataclassesrpathlibrtypingrrrrrBtorch.utils.checkpointr generationr modeling_utilsr utilsr rrrrrconfiguration_rwkvr get_loggerrqr%r#r0autogradFunctionr2rrModulerrrrrrrrD__all__rurnr/rbs]  !// )-+   H %5@g %..11g T)XbC5 C5L#)bii#)L D?]/?]?]D ?? ?: ?? ?@o6#o6o6d i )?i  i X Brn