
    Uh                        d dl Z d dlmZmZmZ d dlZd dlmZ ddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-  e*       rd dl.m/Z/ ddl0m1Z1  e+jd                  e3      Z4 G d dejj                        Z6d Z7d=dZ8dejr                  de:dejr                  fdZ;d Z< G d dejj                        Z= G d  d!e=      Z> G d" d#e=      Z? ed$       G d% d&ejj                               Z@e=e>e?d'ZA G d( d)e      ZBe( G d* d+e#             ZC G d, d-ejj                        ZDe( G d. d/eC             ZE G d0 d1ee'      ZFe( G d2 d3eCe             ZG e(d45       G d6 d7eC             ZHe( G d8 d9eC             ZIe( G d: d;eC             ZJg d<ZKy)>    N)OptionalTupleUnion)nn   )ACT2FN)CacheDynamicCacheStaticCache)GenerationMixin)use_kernel_forward_from_hub)AttentionMaskConverter)FlashAttentionKwargs_flash_attention_forward!flash_attn_supports_top_left_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)PreTrainedModel)Unpack)
LossKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging   )DiffLlamaConfig)	BlockMask)make_flex_block_causal_maskc                   $     e Zd Z fdZd Z xZS )DiffLlamaMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        t        |j                     | _        y NFbias)super__init__confighidden_sizeintermediate_sizer   Linear	gate_projup_proj	down_projr   
hidden_actact_fnselfr-   	__class__s     /var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/diffllama/modeling_diffllama.pyr,   zDiffLlamaMLP.__init__A   s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV../    c                     | j                  | j                  | j                  |            | j                  |      z        }|S N)r3   r5   r1   r2   )r7   xr3   s      r9   forwardzDiffLlamaMLP.forwardK   s6    NN4;;t~~a/@#ADLLQRO#ST	r:   )__name__
__module____qualname__r,   r>   __classcell__r8   s   @r9   r&   r&   @   s    0r:   r&   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..N   dim)shapetorchcat)r=   x1x2s      r9   rotate_halfrN   P   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r:   c                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezerN   )qkcossinposition_idsunsqueeze_dimq_embedk_embeds           r9   apply_rotary_pos_embrY   W   sY    ( --
&C
--
&C3w;q>C/0G3w;q>C/0GGr:   hidden_statesn_repreturnc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rI   expandreshape)rZ   r[   batchnum_key_value_headsslenhead_dims         r9   	repeat_kvrd   r   so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr:   c                 >    ddt        j                  d| z        z  z
  S )Ng?g333333?g333333ӿ)mathexp)	layer_idxs    r9   lambda_init_fnri   ~   s     txxy 01111r:   c                   b    e Zd ZdZddedee   f fdZ	 	 	 	 	 	 ddej                  de
ej                  ej                  f   deej                     deej                     d	ee   d
ededeej                     de
ej                  eej                     ee
ej                        f   fdZ xZS )DiffLlamaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr-   rh   c                    t         |           || _        || _        |-t        j                  d| j                  j                   d       |j                  | _        |j                  | _	        |j                  | _        t        |d| j                  | j                  z        | _        |j                  | _        | j                  | j                  z  | _        |j                   | _        |j"                  | _        d| _        t'        j(                  | j                  | j                  | j                  z  |j*                        | _        t'        j(                  | j                  | j                  | j                  z  |j*                        | _        t'        j(                  | j                  | j                  | j                  z  |j*                        | _        t'        j(                  | j                  | j                  z  | j                  |j*                        | _        t5        |      | _        t'        j8                  t;        j<                  d|j>                  | j                  f            | _         t'        j8                  t;        j<                  d|j>                  | j                  f            | _!        t'        j8                  t;        j<                  d|j>                  | j                  f            | _"        t'        j8                  t;        j<                  d|j>                  | j                  f            | _#        t'        jH                  d| j                  z  |jJ                  d	
      | _&        y )NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.rc   Tr)   r   )sizerF   F)epselementwise_affine)'r+   r,   r-   rh   loggerwarning_oncer8   r?   attention_dropoutr.   num_attention_heads	num_headsgetattrrc   ra   num_key_value_groupsmax_position_embeddings
rope_theta	is_causalr   r0   attention_biasq_projk_projv_projo_projri   lambda_init	ParameterrJ   normallambda_std_dev	lambda_q1	lambda_k1	lambda_q2	lambda_k2RMSNormrms_norm_eps	groupnormr7   r-   rh   r8   s      r9   r,   zDiffLlamaAttention.__init__   s~   " !8!8 9 :, , "(!9!9!--33
D4D4D4VW#)#=#= $(NNd6N6N$N!'-'E'E$ ++ii 0 0$..4==2PW]WlWlmii 0 0$2J2JT]]2Zagavavwii 0 0$2J2JT]]2Zagavavwii >@P@PW]WlWlm))4ell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdA$56;N;Nchir:   rZ   position_embeddingsattention_maskrU   past_key_valueoutput_attentions	use_cachecache_positionr\   c	                 $   |j                         \  }
}}|}| j                  |      }| j                  |      }| j                  |      }|j	                  |
|| j
                  | j                        j                  dd      }|j	                  |
|| j                  | j                        j                  dd      }|j	                  |
|| j                  | j                        j                  dd      }|\  }}t        ||||      \  }}|'|||d}|j                  ||| j                  |      \  }}t        || j                        }t        || j                        }t        j                  t        j                   |dd      d      }|j#                  dddd      }t        j$                  ||j                  dd            t'        j(                  | j                        z  }|#|d d d d d d d |j*                  d   f   }||z   }t,        j.                  j1                  |dt        j2                        j5                  |j6                        }t,        j.                  j9                  || j:                  | j<                  	      }t        j>                  t        j@                  | jB                  | jD                  z  dt        j2                              j5                  |j6                        }t        j>                  t        j@                  | jF                  | jH                  z  dt        j2                              j5                  |j6                        }||z
  | jJ                  z   }t        j$                  ||      }t        j                   |dd      \  }}|||z  z
  }d| jJ                  z
  | jM                  |      z  }|j                  dd      jO                         }|jQ                  |
|d      }| jS                  |      }|sd }||fS )
Nr!   rF   rT   rS   r   rG   rE   r   rH   dtype)ptraining)*rm   r{   r|   r}   viewrt   rc   	transposera   rY   updaterh   rd   rv   rJ   rK   chunkrepeatmatmulrf   sqrtrI   r   
functionalsoftmaxfloat32tor   dropoutrr   r   rg   sumr   r   r   r   r   r   
contiguousr_   r~   )r7   rZ   r   r   rU   r   r   r   r   kwargsbsz
target_len_q_lenquery_states
key_statesvalue_statesrS   rT   cache_kwargsattn_weightscausal_masklambda_1lambda_2lambda_fullattn_outputattn_output1attn_output2s                               r9   r>   zDiffLlamaAttention.forward   s    +//1Z{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm&S#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$Jz4+D+DE
 t/H/HIyy\1!!D"M#**1aA6||L*2F2Fq!2LMPTPYPYZ^ZgZgPhh%(Aq2HJ4D4DR4H2H)HIK'+5L }},,\r,WZZ[g[m[mn}},,\T=S=S^b^k^k,l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<ll<>%*[[aQ%G"l"[<%??4+++t~~k/JJ!++Aq1<<>!))#ub9kk+. LL((r:   r<   NNNFFN)r?   r@   rA   __doc__r"   r   intr,   rJ   Tensorr   
LongTensorr	   boolr>   rB   rC   s   @r9   rk   rk      s    G j  j8C=  jL 2637*."'59B)||B) #5<<#=>B) !.	B)
 u//0B) !B)  B) B) !!1!12B) 
u||Xell3XeELL>Q5RR	SB)r:   rk   c                   P    e Zd ZdZ fdZ	 	 	 	 	 	 ddej                  deej                  ej                  f   deej                     deej                     dee
   ded	ed
eej                     deej                  eej                     eeej                        f   fdZ xZS )DiffLlamaFlashAttention2aN  
    DiffLlama flash attention module. This module inherits from `DiffLlamaAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 B    t        |   |i | t               | _        y r<   )r+   r,   r   _flash_attn_uses_top_left_mask)r7   argsr   r8   s      r9   r,   z!DiffLlamaFlashAttention2.__init__   s#    $)&)
 /P.Q+r:   rZ   r   r   rU   r   r   r   r   r\   c	                 	   t        |t              rt        d      d}|j                         \  }	}
}| j	                  |      }| j                  |      }| j                  |      }|j                  |	|
| j                  | j                        j                  dd      }|j                  |	|
| j                  | j                        j                  dd      }|j                  |	|
| j                  | j                        j                  dd      }|+t        j                  d       | j                  ||      \  }}n|\  }}t        ||||      \  }}|'|||d}|j!                  ||| j"                  |      \  }}|j                  dd      }|j                  dd      }|j                  dd      }| j$                  r| j&                  nd}|j(                  }|t*        j,                  k(  rt+        j.                         rt+        j0                         }nMt3        | j4                  d      r| j4                  j6                  }n | j                  j8                  j(                  }t        j                  d	| d
       |j;                  |      }|j;                  |      }|j;                  |      }t+        j<                  |dd      \  }}|j?                  dddd      }|j?                  dddd      }tA        |||||
||tC        | dd       | jD                  | jF                  
      }tA        |||||
||tC        | dd       | jD                  | jF                  
      }t+        jH                  ||gd      }t+        j<                  |dd      \  }}t+        jJ                  t+        jL                  | jN                  | jP                  z  dt*        j,                              j;                  |j(                        }t+        jJ                  t+        jL                  | jR                  | jT                  z  dt*        j,                              j;                  |j(                        }||z
  | jV                  z   }|||z  z
  }d| jV                  z
  | jY                  |      z  }|j[                  |	|
d      j]                         }| j_                  |      }|sd }|fS )Nz`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformersFr!   rF   aY  The attention layers in this model are transitioning from computing the RoPE embeddings internally through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed `position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be removed and `position_embeddings` will be mandatory.r           _pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .rG   sliding_window)rU   r   r   use_top_left_maskry   rE   r   )0
isinstancer   
ValueErrorrm   r{   r|   r}   r   rt   rc   r   ra   rp   rq   
rotary_embrY   r   rh   r   rr   r   rJ   r   is_autocast_enabledget_autocast_gpu_dtypehasattrr-   r   weightr   r   r   r   ru   r   ry   rK   rg   r   r   r   r   r   r   r   r_   r   r~   )r7   rZ   r   r   rU   r   r   r   r   r   r   r   r   r   r   rS   rT   r   dropout_rateinput_dtypetarget_dtypevalue_states1value_states2r   r   r   r   r   r   r   s                                 r9   r>   z DiffLlamaFlashAttention2.forward   sh    nk2} 
 "%**,UA{{=1[[/
{{=1
 $((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm&G |\BHC*HC#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$J $--a3))!Q/
#--a315t--C #((%--'((*$;;=&?@#{{BB#{{1177 >$ (??<8L#|4J'??<8L',{{<'J$}%,,Q1a8%,,Q1a8/% "4)94@"AAnn
 0% "4)94@"AAnn
 ii| <"E%*[[aQ%G"l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<"[<%??4+++t~~k/JJ!))#ub9DDFkk+. LL((r:   r   )r?   r@   rA   r   r,   rJ   r   r   r   r   r	   r   r>   rB   rC   s   @r9   r   r      s    R 6:37*."'59D)||D) #5<<#=>D) !!1!12	D)
 u//0D) !D)  D) D) !!1!12D) 
u||Xell3XeELL>Q5RR	SD)r:   r   c                   J    e Zd ZdZ	 	 	 	 	 	 ddej
                  deej
                  ej
                  f   deej
                     deej                     dee	   de
de
d	eej                     d
eej
                  eej
                     eeej
                        f   f fdZ xZS )DiffLlamaSdpaAttentiona   
    DiffLlama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
    `DiffLlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
    SDPA API.
    rZ   r   r   rU   r   r   r   r   r\   c	           
         |r,t         j                  d       t        |   ||||||||      S |j	                         \  }
}}| j                  |      }| j                  |      }| j                  |      }|j                  |
|| j                  | j                        j                  dd      }|j                  |
|| j                  | j                        j                  dd      }|j                  |
|| j                  | j                        j                  dd      }|\  }}t        ||||      \  }}|'|||d}|j                  ||| j                  |      \  }}t!        || j"                        }t!        || j"                        }t%        j&                  t%        j(                  |dd      d      }|j+                  dddd      }|}||d d d d d d d |j,                  d   f   }|j.                  j0                  d	k(  r2|0|j3                         }|j3                         }|j3                         }||dkD  rd
nd}t$        j4                  j6                  j9                  ||||| j:                  r| j<                  nd|      }t%        j(                  |dd      \  }}t%        j>                  t%        j@                  | jB                  | jD                  z  dt$        jF                              jI                  |jJ                        }t%        j>                  t%        j@                  | jL                  | jN                  z  dt$        jF                              jI                  |jJ                        }||z
  | jP                  z   }|||z  z
  }d| jP                  z
  | jS                  |      z  }|j                  dd      j3                         }|j                  |
|d      }| jU                  |      }|d fS )Na  DiffLlamaModel is using DiffLlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rZ   r   rU   r   r   r   r   r   r!   rF   r   rG   rE   r   cudaTFr   )	attn_mask	dropout_pry   r   )+rp   rq   r+   r>   rm   r{   r|   r}   r   rt   rc   r   ra   rY   r   rh   rd   rv   rJ   rK   r   r   rI   devicetyper   r   r   scaled_dot_product_attentionr   rr   rg   r   r   r   r   r   r   r   r   r   r   r~   )r7   rZ   r   r   rU   r   r   r   r   r   r   r   r   r   r   r   rS   rT   r   r   ry   r   r   r   r   r   r   r8   s                              r9   r>   zDiffLlamaSdpaAttention.forward  s    [ 7?+-)-"3#-$7 # 	 	 &**,UA{{=1[[/
{{=1#((eT^^T]]S]]^_abc__S%1I1I4==Yccdeghi
#((eT5M5Mt}}]gghiklm&S#7jRUWZ#[ j%#&snUL'5'<'<ZW[WeWegs't$Jz4+D+DE
 t/H/HIyy\1!!D"M#**1aA6$%%aA/E1A1A"1E/E&EFK ##v-+2I'224L#..0J'224L (/EAID5	hh))FF!04d,,3 G 
 &+[[aQ%G"l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<"[<%??4+++t~~k/JJ!++Aq1<<>!&&sE26kk+.D  r:   r   )r?   r@   rA   r   rJ   r   r   r   r   r	   r   r>   rB   rC   s   @r9   r   r     s     2637*."'59\!||\! #5<<#=>\! !.	\!
 u//0\! !\!  \! \! !!1!12\! 
u||Xell3XeELL>Q5RR	S\! \!r:   r   r   c                   ,     e Zd Zd fd	Zd Zd Z xZS )DiffLlamaRMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z?
        DiffLlamaRMSNorm is equivalent to T5LayerNorm
        N)r+   r,   r   r   rJ   onesr   variance_epsilon)r7   r.   rn   r8   s      r9   r,   zDiffLlamaRMSNorm.__init__  s1     	ll5::k#:; #r:   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )NrF   rE   T)keepdim)	r   r   rJ   r   powmeanrsqrtr   r   )r7   rZ   r   variances       r9   r>   zDiffLlamaRMSNorm.forward  sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r:   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler   rI   r   r7   s    r9   
extra_reprzDiffLlamaRMSNorm.extra_repr  s*    ))*+6$2G2G1HIIr:   )gư>)r?   r@   rA   r,   r>   r   rB   rC   s   @r9   r   r     s    $;Jr:   r   )eagerflash_attention_2sdpac                   p    e Zd Zdedef fdZ	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
   dee   d	ee   d
eej                     deeej                  ej                  f      dee   deej                  eeej                  ej                  f      f   fdZ xZS )DiffLlamaDecoderLayerr-   rh   c                 :   t         |           |j                  | _        t        |j                     ||      | _        t        |      | _        t        |j                  |j                        | _
        t        |j                  |j                        | _        y )N)r-   rh   rn   )r+   r,   r.   DIFFLLAMA_ATTENTION_CLASSES_attn_implementation	self_attnr&   mlpr   r   input_layernormpost_attention_layernormr   s      r9   r,   zDiffLlamaDecoderLayer.__init__  sz    !--4V5P5PQY_ktu'/0B0BH[H[\(89K9KQWQdQd(e%r:   rZ   r   rU   r   r   r   r   r   r   r\   c	                     |}
| j                  |      } | j                  d||||||||d|	\  }}|
|z   }|}
| j                  |      }| j                  |      }|
|z   }|f}|r||fz  }|S )Nr    )r   r   r   r   )r7   rZ   r   rU   r   r   r   r   r   r   residualself_attn_weightsoutputss                r9   r>   zDiffLlamaDecoderLayer.forward  s     !,,]; ,:4>> 
,
')%)/) 3
,
 
,
(( !=0 !55mD/ =0 ")++Gr:   )NNNFFNN)r?   r@   rA   r"   r   r,   rJ   r   r   r   r	   r   r   r   r   FloatTensorr>   rB   rC   s   @r9   r   r     s   f f3 f 2637*.,1$)59KO'||' !.' u//0	'
 !' $D>' D>' !!1!12' &eELL%,,,F&GH' -.' 
u  (51B1BEDUDU1U+V"WW	X'r:   r   c                   F    e Zd ZeZdZdZdgZdgZdZ	dZ
dZdZdZdZdZd Zy)DiffLlamaPreTrainedModelmodelTr   past_key_valuesFc                 t   | j                   j                  }t        |t        j                        rY|j
                  j                  j                  d|       |j                  %|j                  j                  j                          y y t        |t        j                        rf|j
                  j                  j                  d|       |j                  2|j
                  j                  |j                     j                          y y t        |t              r&|j
                  j                  j                  d       y t        |t              r|j                  j                  j                  d| j                   j                          |j"                  j                  j                  d| j                   j                          |j$                  j                  j                  d| j                   j                          |j&                  j                  j                  d| j                   j                          y y )Nr   )r   stdg      ?r   )r-   initializer_ranger   r   r0   r   datanormal_r*   zero_	Embeddingpadding_idxr   fill_rk   r   r   r   r   r   )r7   moduler  s      r9   _init_weightsz&DiffLlamaPreTrainedModel._init_weightsI  s   kk++fbii(MM&&CS&9{{&  &&( '-MM&&CS&9!!-""6#5#56<<> . 01MM$$S) 23!!))!T[[-G-GH!!))!T[[-G-GH!!))!T[[-G-GH!!))!T[[-G-GH	 4r:   N)r?   r@   rA   r"   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_flex_attn_supports_cache_class_supports_quantized_cache_supports_static_cache_supports_attention_backendr  r   r:   r9   r  r  :  sT    "L&*#01#4"5!N  $!"'Ir:   r  c                   ^     e Zd Zddef fdZ ej                         ed               Z xZ	S )DiffLlamaRotaryEmbeddingr-   c                    t         |           t        |d      rG|j                  ;|j                  j	                  d|j                  j	                  d            | _        nd| _        |j                  | _        |j                  | _        || _	        t        | j
                     | _        | j                  | j                  |      \  }| _        | j                  d|d       | j                  | _        y )Nrope_scaling	rope_typer   defaultinv_freqF)
persistent)r+   r,   r   r  getr  rw   max_seq_len_cachedoriginal_max_seq_lenr-   r   rope_init_fnattention_scalingregister_bufferr!  original_inv_freq)r7   r-   r   r!  r8   s       r9   r,   z!DiffLlamaRotaryEmbedding.__init__]  s    6>*v/B/B/N#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q($(ZeD!%r:   c                 b   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        j                  |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   rE   r!   mpscpuF)device_typeenabledrF   rG   )r   )r!  floatr^   rI   r   r   r   r   strrJ   autocastr   rK   rS   r'  rT   r   )
r7   r=   rU   inv_freq_expandedposition_ids_expandedr-  freqsembrS   rT   s
             r9   r>   z DiffLlamaRotaryEmbedding.forwardn  sV    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk^^UC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s    BF%%F.r<   )
r?   r@   rA   r"   r,   rJ   no_gradr   r>   rB   rC   s   @r9   r  r  \  s3    / /" U]]_<  <r:   r  c                       e Zd Zdef fdZd Zd Zee	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e   d	e	e
j                     d
e	e   de	e   de	e   de	e
j                     dee   defd              Z	 ddee
j                  df   de
j                  de
j                  dedef
dZede
j                  dedede
j0                  de
j                  defd       Z xZS )DiffLlamaModelr-   c           	         t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t#        |      | _        d| _        | j)                          y c c}w )Nr   )r-   F)r+   r,   pad_token_idr  
vocab_sizer   r
  r.   embed_tokens
ModuleListrangenum_hidden_layersr   layersr   r   normr  r   gradient_checkpointing	post_initr   s      r9   r,   zDiffLlamaModel.__init__  s     !.. ++LL):):F<N<NPTP`P`ammGLVMeMeGfg)"695g
 %V%7%7V=P=PQ	2&A&+# 	 hs   Dc                     | j                   S r<   r<  r   s    r9   get_input_embeddingsz#DiffLlamaModel.get_input_embeddings  s       r:   c                     || _         y r<   rE  r7   values     r9   set_input_embeddingsz#DiffLlamaModel.set_input_embeddings  s
    !r:   	input_idsr   rU   r  inputs_embedsr   r   output_hidden_statesr   flash_attn_kwargsr\   c
                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|d u |d uz  rt	        d      | j
                  r%| j                  r|rt        j                  d       d}t        |t        d       t        f      st	        d      || j                  |      }|r|
t               }|	F||j                         nd}t        j                   |||j"                  d   z   |j$                        }	||	j'                  d      }| j)                  |||	||      }|}| j+                  ||      }|rdnd }|rdnd }| j,                  d | j                   j.                   D ],  }|r||fz  } ||f||||||	|d	|
}|d   }|s$||d   fz  }. | j1                  |      }|r||fz  }t3        ||r|nd ||
      S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBThe `past_key_values` should be either a `Cache` object or `None`.r   r!   r   r   )r   rU   r   r   r   r   r   )last_hidden_stater  rZ   
attentions)r-   r   rM  r   r   rB  r   rp   rq   r   r   r	   r<  r
   get_seq_lengthrJ   arangerI   r   rP   _update_causal_maskr   r@  r?  rA  r   )r7   rK  r   rU   r  rL  r   r   rM  r   rN  past_seen_tokensr   rZ   r   all_hidden_statesall_self_attnsdecoder_layerlayer_outputss                      r9   r>   zDiffLlamaModel.forward  sT    2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==Yj I /DJ+>?abb  --i8M0*nO!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6L..M>?L]
 & #oom\J #7BD0d![[)H4;;+H+HI 	6M#!m%55!)
*)."3#-$7
 $
M *!,M =#3"55'	6* 		-0  -!11&+/8Od+%	
 	
r:   r#   input_tensorc           	         | j                   j                  dk(  r||dk(  j                         r|S y | j                   j                  dk(  r't        |t        j
                        rt        |      }|S ||j                         nd}||j                  nd}| j                   j                  dk(  r(|s&|s$t        j                  |||| j                        ry |j                  }|j                  d   }	|r|j                         }
n1t        |t        j
                        r|j                  d	   n||	z   dz   }
| j                  ||	|
|||j                  d   
      }| j                   j                  dk(  rQ|O|j                   j"                  dv r7|s5t	        j$                  |      j&                  }t        j(                  ||      }|S )Nr   r   flex_attentionr   Fr   )rL  past_key_values_lengthis_trainingr!   rE   )sequence_lengthtarget_lengthr   r   
batch_size)r   xpunpu)r-   r   anyr   rJ   r   r$   rS  is_compileabler   _ignore_causal_mask_sdpar   r   rI   get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr   r   finfomin_unmask_unattended)r7   r   r[  r   r  r   rV  using_compilable_cacher   r`  ra  r   	min_dtypes                r9   rU  z"DiffLlamaModel._update_causal_mask  s    ;;++/BB)~/D.I.I.K%%;;++/??.%,,7!<^!L!!
 @O?Z?99;`aCRC^!?!?di ;;++v5>T]n%>>*'7 MM	 ""&,,Q/!+??AM nell; $$R(%7!;  PP+')#))!, Q 
 KK,,6*%%**.DD%
 E*..I0CCKQZ[Kr:   r`  ra  r   rb  c                    | | j                         dk(  r| }|S t        j                  |      j                  }t        j                  ||f|||j
                        }|dk7  rt        j                  |d      }|t        j                  ||j
                        |j                  dd      kD  z  }|ddddddf   j                  |ddd      }| |j                         }| j                  d   }	|ddddddd|	f   | ddddddf   j                  |j
                        z   }
|
dk(  }
|ddddddd|	f   j                  |
|      |ddddddd|	f<   |S )	aM  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        N   )
fill_valuer   r   r!   )diagonalrP  rE   r   )rH   rJ   rj  rk  fullr   triurT  r_   r^   clonerI   r   masked_fill)r   r`  ra  r   r   rb  r   r   rn  mask_lengthpadding_masks              r9   ri  zDDiffLlamaModel._prepare_4d_causal_attention_mask_with_cache_position:  s   < %.*<*<*>!*C(K* ' E*..I** -0Ye\j\q\qK !##jjqA5<<n>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c )6Aq!\k\12 r:   	NNNNNNNNN)F)r?   r@   rA   r"   r,   rF  rJ  r   r   r   rJ   r   r   r	   r   r   r   r   r   r>   r   rU  staticmethodr   r   ri  rB   rC   s   @r9   r8  r8  ~  s     !"  151537+/59$(,0/359\
E,,-\
 !.\
 u//0	\

 "%\
   1 12\
 D>\
 $D>\
 'tn\
 !!1!12\
 $$89\
 
!\
  \
H #(BellK78B llB 	B
 B  BH 444 4 {{	4
 4 4 4r:   r8  c                       e Zd Zy)KwargsForCausalLMN)r?   r@   rA   r   r:   r9   r|  r|  r  s    r:   r|  c                       e Zd ZdgZddiZddgdgfiZ fdZd Zd Zd	 Z	d
 Z
d Zd Zee	 	 	 	 	 	 	 	 	 	 	 ddeej"                     deej$                     deej"                     dee   deej(                     deej"                     dee   dee   dee   deej"                     deeej$                  f   dee   defd              Z xZS )DiffLlamaForCausalLMzlm_head.weightlm_headcolwise_reprZ   logitsc                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        | j                          y r(   )
r+   r,   r8  r  r;  r   r0   r.   r  rC  r6   s     r9   r,   zDiffLlamaForCausalLM.__init__{  sU     #F+
 ++yy!3!3V5F5FUS 	r:   c                 .    | j                   j                  S r<   r  r<  r   s    r9   rF  z)DiffLlamaForCausalLM.get_input_embeddings      zz&&&r:   c                 &    || j                   _        y r<   r  rH  s     r9   rJ  z)DiffLlamaForCausalLM.set_input_embeddings      "'

r:   c                     | j                   S r<   r  r   s    r9   get_output_embeddingsz*DiffLlamaForCausalLM.get_output_embeddings  s    ||r:   c                     || _         y r<   r  )r7   new_embeddingss     r9   set_output_embeddingsz*DiffLlamaForCausalLM.set_output_embeddings  s	    %r:   c                     || _         y r<   r  )r7   decoders     r9   set_decoderz DiffLlamaForCausalLM.set_decoder  s	    
r:   c                     | j                   S r<   r  r   s    r9   get_decoderz DiffLlamaForCausalLM.get_decoder  s    zzr:   rK  r   rU   r  rL  labelsr   r   rM  r   logits_to_keepr   r\   c                    ||n| j                   j                  }|	|	n| j                   j                  }	 | j                  d||||||||	|
d	|}|j                  }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}|* | j                  d||| j                   j                  d|}t        |||j                  |j                  |j                        S )a1  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, DiffLlamaForCausalLM

        >>> model = DiffLlamaForCausalLM.from_pretrained("google/diffllama-7b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/diffllama-7b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```N)	rK  r   rU   r  rL  r   r   rM  r   )r  r  r;  lossr  r  rZ   rR  r   )r-   r   rM  r  rQ  r   r   slicer  loss_functionr;  r   r  rZ   rR  )r7   rK  r   rU   r  rL  r  r   r   rM  r   r  r   r   rZ   slice_indicesr  r  s                     r9   r>   zDiffLlamaForCausalLM.forward  s   N 2C1N-TXT_T_TqTq$8$D $++JjJj 	
 ,64:: ,
)%+'/!5),
 ,
  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD%#33!//))
 	
r:   )NNNNNNNNNNr   )r?   r@   rA   _tied_weights_keys_tp_plan_pp_planr,   rF  rJ  r  r  r  r  r   r   r   rJ   r   r   r	   r   r   r   r   r   r|  r   r>   rB   rC   s   @r9   r~  r~  u  s   *+=)H_-z:;H'(&  151537+/59-1$(,0/35934G
E,,-G
 !.G
 u//0	G

 "%G
   1 12G
 ))*G
 D>G
 $D>G
 'tnG
 !!1!12G
 c5<</0G
 *+G
 
 G
  G
r:   r~  a  
    The DiffLlama Model transformer with a sequence classification head on top (linear layer).

    [`DiffLlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )custom_introc                       e Zd Z fdZd Zd Zee	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   dee	j                     d	ee	j                     d
ee   dee   dee   defd              Z xZS )"DiffLlamaForSequenceClassificationc                     t         |   |       |j                  | _        t        |      | _        t        j                  |j                  | j                  d      | _        | j                          y r(   )
r+   r,   
num_labelsr8  r  r   r0   r.   scorerC  r6   s     r9   r,   z+DiffLlamaForSequenceClassification.__init__  sS      ++#F+
YYv114??O
 	r:   c                 .    | j                   j                  S r<   r  r   s    r9   rF  z7DiffLlamaForSequenceClassification.get_input_embeddings  r  r:   c                 &    || j                   _        y r<   r  rH  s     r9   rJ  z7DiffLlamaForSequenceClassification.set_input_embeddings  r  r:   rK  r   rU   r  rL  r  r   r   rM  r\   c
           
         | j                  ||||||||	      }
|
j                  }| j                  |      }||j                  d   }n|j                  d   }| j                  j
                  |dk7  rt        d      | j                  j
                  d}n||| j                  j
                  k7  j                  |j                  t        j                        }t        j                  |j                  d   |j                  t        j                        }||z  j                  d      }n.d}t        j                  | j                  j                    d       |t        j                  ||j                  	      |f   }d}|| j#                  |||| j                  
      }t%        |||
j&                  |
j(                  |
j*                        S )  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r   rU   r  rL  r   r   rM  Nr   r!   z=Cannot handle batch sizes > 1 if no padding token is defined.rE   )r   r   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rP  )r  r  pooled_logitsr-   r  )r  rQ  r  rI   r-   r:  r   r   r   rJ   int32rT  argmaxrp   rq   r8   r?   r  r   r  rZ   rR  )r7   rK  r   rU   r  rL  r  r   r   rM  transformer_outputsrZ   r  rb  last_non_pad_tokennon_pad_masktoken_indicesr  r  s                      r9   r>   z*DiffLlamaForSequenceClassification.forward   s   * 8<zz)%+'/!5 8B 	8
 ,==M* "+J&,,Q/J;;##+
a\]];;##+!#"%)A)AAEEfmmUZU`U`aL!LL)<V]]Z_ZeZefM"/,">!F!Fr!J!#>>**+ ,Z Z
 u||Jv}}MOaab%%VFR_hlhshs%tD/ /??-;;*55
 	
r:   ry  )r?   r@   rA   r,   rF  rJ  r   r   r   rJ   r   r   r	   r   r   r   r>   rB   rC   s   @r9   r  r    s    '(  151537+/59-1$(,0/3A
E,,-A
 !.A
 u//0	A

 "%A
   1 12A
 ))*A
 D>A
 $D>A
 'tnA
 
*A
  A
r:   r  c                   4    e Zd ZdZ fdZd Zd Zee	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e   d	e	e
j                     d
e	e
j                     de	e
j                     de	e   de	e   defd              Z xZS )DiffLlamaForQuestionAnsweringtransformerc                     t         |   |       t        |      | _        t	        j
                  |j                  d      | _        | j                          y )NrF   )	r+   r,   r8  r  r   r0   r.   
qa_outputsrC  r6   s     r9   r,   z&DiffLlamaForQuestionAnswering.__init__J  sA     )&1))F$6$6: 	r:   c                 .    | j                   j                  S r<   r  r<  r   s    r9   rF  z2DiffLlamaForQuestionAnswering.get_input_embeddingsR  s    ,,,r:   c                 &    || j                   _        y r<   r  rH  s     r9   rJ  z2DiffLlamaForQuestionAnswering.set_input_embeddingsU  s    (-%r:   rK  r   rU   r  rL  start_positionsend_positionsr   rM  r\   c
           	         | j                  |||||||	      }|j                  }| j                  |      }|j                  dd      \  }}|j	                  d      j                         }|j	                  d      j                         }d }|| | j                  ||||fi |
}t        ||||j                  |j                        S )N)r   rU   r  rL  r   rM  r!   rE   rG   )r  start_logits
end_logitsrZ   rR  )
r  rQ  r  splitsqueezer   r  r   rZ   rR  )r7   rK  r   rU   r  rL  r  r  r   rM  r   r   sequence_outputr  r  r  r  s                    r9   r>   z%DiffLlamaForQuestionAnswering.forwardX  s     ,0+;+;)%+'/!5 ,< ,
 "331#)<<r<#: j#++B/::<''+668
&=+D%4%%lJQ^ibhiD+%!!//))
 	
r:   ry  )r?   r@   rA   r  r,   rF  rJ  r   r   r   rJ   r   r   r	   r   r   r   r>   rB   rC   s   @r9   r  r  F  s    %-.  151537+/596:48,0/3(
E,,-(
 !.(
 u//0	(

 "%(
   1 12(
 "%"2"23(
   0 01(
 $D>(
 'tn(
 
&(
  (
r:   r  c                       e Zd Z fdZd Zd Zee	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   dee	j                     d	ee	j                     d
ee   dee   dee   defd              Z xZS )DiffLlamaForTokenClassificationc                    t         |   |       |j                  | _        t        |      | _        t        |dd       |j                  }nt        |dd       |j                  }nd}t        j                  |      | _
        t        j                  |j                  |j                        | _        | j                          y )Nclassifier_dropouthidden_dropoutg?)r+   r,   r  r8  r  ru   r  r  r   Dropoutr   r0   r.   r  rC  )r7   r-   r  r8   s      r9   r,   z(DiffLlamaForTokenClassification.__init__  s      ++#F+
6/6B!'!:!:V-t4@!'!6!6!$zz"45YYv1163D3DE
 	r:   c                 .    | j                   j                  S r<   r  r   s    r9   rF  z4DiffLlamaForTokenClassification.get_input_embeddings  r  r:   c                 &    || j                   _        y r<   r  rH  s     r9   rJ  z4DiffLlamaForTokenClassification.set_input_embeddings  r  r:   rK  r   rU   r  rL  r  r   r   rM  r\   c
           
         | j                  ||||||||	      }
|
j                  }| j                  |      }| j                  |      }d}|| j	                  ||| j
                        }t        |||
j                  |
j                        S )r  r  N)r  r  rZ   rR  )	r  rQ  r   r  r  r-   r   rZ   rR  )r7   rK  r   rU   r  rL  r  r   r   rM  r   r  r  r  s                 r9   r>   z'DiffLlamaForTokenClassification.forward  s    * ,0::)%+'/!5 ,6 	,
 "33,,7O,%%ffdkkBD$!//))	
 	
r:   ry  )r?   r@   rA   r,   rF  rJ  r   r   r   rJ   r   r   r	   r   r   r   r>   rB   rC   s   @r9   r  r    s     '(  151537+/59-1$(,0/3*
E,,-*
 !.*
 u//0	*

 "%*
   1 12*
 ))**
 D>*
 $D>*
 'tn*
 
*
  *
r:   r  )r  r8  r~  r  r  r  )Nr!   )Lrf   typingr   r   r   rJ   r   activationsr   cache_utilsr	   r
   r   
generationr   integrationsr   modeling_attn_mask_utilsr   modeling_flash_attention_utilsr   r   r   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   r    configuration_diffllamar"   !torch.nn.attention.flex_attentionr#   integrations.flex_attentionr$   
get_loggerr?   rp   Moduler&   rN   rY   r   r   rd   ri   rk   r   r   r   r   r   r  r  r8  r|  r~  r  r  r  __all__r   r:   r9   <module>r     s%  0  ) )   ! ; ; ) 7 > 
 :  L - & h h 4  !;J 
		H	%299  (6	UU\\ 	U# 	U%,, 	U2g) g)TS)1 S)ld!/ d!N Y'Jryy J (J*  1" 26 2j I I IB<ryy <D p- p pf ?,j > i
3_ i
 i
X S
)A S
S
l ;
$< ;
 ;
| C
&> C
 C
Lr:   