
    Uh                        d Z ddlZddlmZmZmZ ddlZddlZddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlm Z   ejB                  e"      Z#ejH                  Z% G d dejL                        Z' G d dejL                        Z( G d dejL                        Z)de(iZ* G d dejL                        Z+ G d dejL                        Z, G d dejL                        Z- G d dejL                        Z. G d dejL                        Z/ G d d ejL                        Z0 G d! d"ejL                        Z1 G d# d$ejL                        Z2 G d% d&ejL                        Z3e G d' d(e             Z4e G d) d*e4             Z5e G d+ d,e4             Z6 ed-.       G d/ d0e4             Z7 ed1.       G d2 d3e4             Z8e G d4 d5e4             Z9g d6Z:y)7zPyTorch LayoutLM model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging   )LayoutLMConfigc                   4     e Zd ZdZ fdZ	 	 	 	 	 ddZ xZS )LayoutLMEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    t         t        |           t        j                  |j
                  |j                  |j                        | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                   |j                        | _        t%        |j                  |j&                        | _        t        j*                  |j,                        | _        | j1                  dt3        j4                  |j                        j7                  d      d       y )N)padding_idxepsposition_ids)r   F)
persistent)superr   __init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsmax_2d_position_embeddingsx_position_embeddingsy_position_embeddingsh_position_embeddingsw_position_embeddingstype_vocab_sizetoken_type_embeddingsLayoutLMLayerNormlayer_norm_eps	LayerNormDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandselfconfig	__class__s     /var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/layoutlm/modeling_layoutlm.pyr$   zLayoutLMEmbeddings.__init__1   s^    $02!||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2S2SU[UgUg%h"%'\\&2S2SU[UgUg%h"%'\\&2S2SU[UgUg%h"%'\\&2S2SU[UgUg%h"%'\\&2H2H&J\J\%]"*6+=+=6CXCXYzz&"<"<=ELL)G)GHOOPWXej 	 	
    c                    ||j                         }n|j                         d d }|d   }||j                  n|j                  }|| j                  d d d |f   }|&t        j                  |t        j
                  |      }|| j                  |      }|}	| j                  |      }
	 | j                  |d d d d df         }| j                  |d d d d df         }| j                  |d d d d df         }| j                  |d d d d df         }| j                  |d d d d df   |d d d d df   z
        }| j                  |d d d d df   |d d d d df   z
        }| j                  |      }|	|
z   |z   |z   |z   |z   |z   |z   |z   }| j                  |      }| j                  |      }|S # t        $ r}t        d      |d }~ww xY w)Nr!   r   dtypedevicer      r
   z:The `bbox`coordinate values should be within 0-1000 range.)sizerF   r    r:   zeroslongr)   r+   r-   r.   
IndexErrorr/   r0   r2   r5   r8   )r>   	input_idsbboxtoken_type_idsr    inputs_embedsinput_shape
seq_lengthrF   words_embeddingsr+   left_position_embeddingsupper_position_embeddingsright_position_embeddingslower_position_embeddingser/   r0   r2   
embeddingss                       rA   forwardzLayoutLMEmbeddings.forwardB   s1     #..*K',,.s3K ^
%.%:!!@T@T,,Q^<L!"[[EJJvVN  00;M("66|D	b'+'A'A$q!Qw-'P$(,(B(B41a=(Q%(,(B(B41a=(Q%(,(B(B41a=(Q% !% : :41a=4PQSTVWPW=;X Y $ : :41a=4PQSTVWPW=;X Y $ : :> J !"&' (( (	(
 (( $$ $$ $$ 	 ^^J/
\\*-
)  	bYZ`aa	bs   ,A,F7 7	G GG)NNNNN)__name__
__module____qualname____doc__r$   rY   __classcell__r@   s   @rA   r   r   .   s!    Q
& 5rB   r   c                   P    e Zd Zd fd	Zdej
                  dej
                  fdZ	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     d	eej                     d
ee	e	ej                           dee
   de	ej
                     fdZ xZS )LayoutLMSelfAttentionc                    t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                        | _        |xs t#        |dd      | _        | j$                  dk(  s| j$                  d	k(  rF|j&                  | _        t        j(                  d
|j&                  z  dz
  | j                        | _        |j,                  | _        y )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsoluterelative_keyrelative_key_queryrG   r   )r#   r$   r'   num_attention_headshasattr
ValueErrorintattention_head_sizeall_head_sizer   Linearquerykeyvaluer6   attention_probs_dropout_probr8   getattrre   r*   r%   distance_embedding
is_decoderr>   r?   re   r@   s      rA   r$   zLayoutLMSelfAttention.__init__|   s    : ::a?PVXhHi#F$6$6#7 8 445Q8 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
zz&"E"EF'> (
'-zC
$ ''>9T=Y=Y]q=q+1+I+ID(&(ll1v7U7U3UXY3Y[_[s[s&tD# ++rB   xreturnc                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )Nr!   r   rG   r   r
   )rH   ri   rm   viewpermute)r>   rx   new_x_shapes      rA   transpose_for_scoresz*LayoutLMSelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$rB   hidden_statesattention_mask	head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsc                 $   | j                  |      }|d u}	|	r||d   }
|d   }|}n |	rC| j                  | j                  |            }
| j                  | j                  |            }|}n|y| j                  | j                  |            }
| j                  | j                  |            }t	        j
                  |d   |
gd      }
t	        j
                  |d   |gd      }n@| j                  | j                  |            }
| j                  | j                  |            }| j                  |      }|d u}| j                  r|
|f}t	        j                  ||
j                  dd            }| j                  dk(  s| j                  dk(  r|j                  d   |
j                  d   }}|rDt	        j                  |dz
  t        j                  |j                  	      j                  dd      }n@t	        j                  |t        j                  |j                  	      j                  dd      }t	        j                  |t        j                  |j                  	      j                  dd      }||z
  }| j!                  || j"                  z   dz
        }|j%                  |j&                  
      }| j                  dk(  rt	        j(                  d||      }||z   }nE| j                  dk(  r6t	        j(                  d||      }t	        j(                  d|
|      }||z   |z   }|t+        j,                  | j.                        z  }|||z   }t0        j2                  j5                  |d      }| j7                  |      }|||z  }t	        j                  ||      }|j9                  dddd      j;                         }|j=                         d d | j>                  fz   }|j                  |      }|r||fn|f}| j                  r||fz   }|S )Nr   r   rG   dimr!   rg   rh   rD   rE   zbhld,lrd->bhlrzbhrd,lrd->bhlrr
   ) rp   r~   rq   rr   r:   catrv   matmul	transposere   shapetensorrJ   rF   r{   r;   ru   r*   torE   einsummathsqrtrm   r   
functionalsoftmaxr8   r|   
contiguousrH   rn   )r>   r   r   r   r   r   r   r   mixed_query_layeris_cross_attention	key_layervalue_layerquery_layer	use_cacheattention_scoresquery_length
key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probscontext_layernew_context_layer_shapeoutputss                               rA   rY   zLayoutLMSelfAttention.forward   s    !JJ}5
 3$>."<&q)I(+K3N11$((;P2QRI33DJJ?T4UVK3N'11$((=2IJI33DJJ}4MNK		>!#4i"@aHI))^A%6$D!LK11$((=2IJI33DJJ}4MNK//0AB"$.	?? (5N !<<Y5H5HR5PQ''>9T=Y=Y]q=q'2'8'8';Y__Q=O*L!&j1nEJJWdWkWk!l!q!q" "'l%**UbUiUi!j!o!oprtu!v"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||<LiYm/n,#36T#TWs#s +dii8P8P.QQ%/.@ --//0@b/I ,,7  -	9O_kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**+BC6G=/2mM]?? 11GrB   NNNNNNF)rZ   r[   r\   r$   r:   Tensorr~   r   FloatTensorr   boolrY   r^   r_   s   @rA   ra   ra   {   s    ,4%ell %u|| % 7;15=A>BDH,1c||c !!2!23c E--.	c
  ((9(9:c !)):): ;c !uU->->'?!@Ac $D>c 
u||	crB   ra   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )LayoutLMSelfOutputc                 (   t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                        | _
        y Nr   )r#   r$   r   ro   r'   denser5   r4   r6   r7   r8   r=   s     rA   r$   zLayoutLMSelfOutput.__init__  s`    YYv1163E3EF
f&8&8f>S>STzz&"<"<=rB   r   input_tensorry   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   r   r8   r5   r>   r   r   s      rA   rY   zLayoutLMSelfOutput.forward	  7    

=1]3}|'CDrB   rZ   r[   r\   r$   r:   r   rY   r^   r_   s   @rA   r   r     1    >U\\  RWR^R^ rB   r   eagerc                       e Zd Zd fd	Zd Z	 	 	 	 	 	 ddej                  deej                     deej                     deej                     deej                     dee	e	ej                           d	ee
   d
e	ej                     fdZ xZS )LayoutLMAttentionc                     t         |           t        |j                     ||      | _        t        |      | _        t               | _        y )Nre   )	r#   r$   LAYOUTLM_SELF_ATTENTION_CLASSES_attn_implementationr>   r   outputsetpruned_headsrw   s      rA   r$   zLayoutLMAttention.__init__  sC    3F4O4OP,C
	 )0ErB   c                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   r   )lenr   r>   ri   rm   r   r   rp   rq   rr   r   r   rn   union)r>   headsindexs      rA   prune_headszLayoutLMAttention.prune_heads  s   u:?749900$))2O2OQUQbQb
u
 -TYY__eD		*499==%@		,TYY__eD		.t{{/@/@%QO )-		(E(EE
(R		%"&))"?"?$))B_B_"_		 --33E:rB   r   r   r   r   r   r   r   ry   c           	      p    | j                  |||||||      }| j                  |d   |      }	|	f|dd  z   }
|
S )Nr   r   )r>   r   )r>   r   r   r   r   r   r   r   self_outputsattention_outputr   s              rA   rY   zLayoutLMAttention.forward1  sW     yy!"
  ;;|AF#%QR(88rB   r   r   )rZ   r[   r\   r$   r   r:   r   r   r   r   r   rY   r^   r_   s   @rA   r   r     s    ";* 7;15=A>BDH,1|| !!2!23 E--.	
  ((9(9: !)):): ; !uU->->'?!@A $D> 
u||	rB   r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )LayoutLMIntermediatec                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r#   r$   r   ro   r'   intermediate_sizer   
isinstance
hidden_actstrr   intermediate_act_fnr=   s     rA   r$   zLayoutLMIntermediate.__init__K  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$rB   r   ry   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r>   r   s     rA   rY   zLayoutLMIntermediate.forwardS  s&    

=100?rB   r   r_   s   @rA   r   r   J  s#    9U\\ ell rB   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )LayoutLMOutputc                 (   t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        y r   )r#   r$   r   ro   r   r'   r   r5   r4   r6   r7   r8   r=   s     rA   r$   zLayoutLMOutput.__init__[  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=rB   r   r   ry   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   r   r   s      rA   rY   zLayoutLMOutput.forwarda  r   rB   r   r_   s   @rA   r   r   Z  r   rB   r   c                       e Zd Z fdZ	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     deej                     deeeej                           dee	   d	eej
                     fd
Z
d Z xZS )LayoutLMLayerc                 f   t         |           |j                  | _        d| _        t	        |      | _        |j                  | _        |j                  | _        | j                  r,| j                  st        |  d      t	        |d      | _	        t        |      | _        t        |      | _        y )Nr   z> should be used as a decoder model if cross attention is addedrf   r   )r#   r$   chunk_size_feed_forwardseq_len_dimr   	attentionrv   add_cross_attentionrk   crossattentionr   intermediater   r   r=   s     rA   r$   zLayoutLMLayer.__init__j  s    '-'E'E$*62 ++#)#=#= ##?? D6)g!hii"3FT^"_D08$V,rB   r   r   r   r   r   r   r   ry   c           	         ||d d nd }| j                  |||||      }	|	d   }
| j                  r|	dd }|	d   }n|	dd  }d }| j                  rT|Rt        | d      st        d|  d      ||d	d  nd }| j	                  |
||||||      }|d   }
||dd z   }|d   }|z   }t        | j                  | j                  | j                  |
      }|f|z   }| j                  r|fz   }|S )
NrG   )r   r   r   r   r!   r   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r   )	r   rv   rj   rk   r   r   feed_forward_chunkr   r   )r>   r   r   r   r   r   r   r   self_attn_past_key_valueself_attention_outputsr   r   present_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputslayer_outputs                    rA   rY   zLayoutLMLayer.forwardx  s}    :H9S>"1#5Y] !%/3 "0 "
 2!4 ??,Qr2G 6r :,QR0G'+$??4@4!12 =dV DD D  @N?Yrs(;_c%&*&9&9 %&)!'#  7q9 7" ==G ,C2+F( 14P P0##T%A%A4CSCSUe
  /G+ ??!2 44GrB   c                 L    | j                  |      }| j                  ||      }|S r   )r   r   )r>   r   intermediate_outputr   s       rA   r   z LayoutLMLayer.feed_forward_chunk  s,    "//0@A{{#68HIrB   r   )rZ   r[   r\   r$   r:   r   r   r   r   r   rY   r   r^   r_   s   @rA   r   r   i  s    -" 7;15=A>BDH,1?||? !!2!23? E--.	?
  ((9(9:? !)):): ;? !uU->->'?!@A? $D>? 
u||	?BrB   r   c                   D    e Zd Z fdZ	 	 	 	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     deej                     deeeej                           dee	   d	ee	   d
ee	   dee	   de
eej
                     ef   fdZ xZS )LayoutLMEncoderc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w )NF)
r#   r$   r?   r   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r>   r?   _r@   s      rA   r$   zLayoutLMEncoder.__init__  sN    ]]5IaIaCb#caM&$9#cd
&+# $ds   A#r   r   r   r   r   past_key_valuesr   r   output_hidden_statesreturn_dictry   c                    |	rdnd }|rdnd }|r| j                   j                  rdnd }| j                  r%| j                  r|rt        j                  d       d}|rdnd }t        | j                        D ]  \  }}|	r||fz   }|||   nd }|||   nd }| j                  r/| j                  r#| j                  |j                  |||||||      }n ||||||||      }|d   }|r	||d   fz  }|s|||d   fz   }| j                   j                  s||d   fz   } |	r||fz   }|
st        d |||||fD              S t        |||||	      S )
N zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   r!   r   rG   c              3   $   K   | ]  }|| 
 y wr   r   ).0vs     rA   	<genexpr>z*LayoutLMEncoder.forward.<locals>.<genexpr>	  s      
 = 
s   )last_hidden_stater   r   
attentionscross_attentions)r?   r   r   trainingloggerwarning_once	enumerater   _gradient_checkpointing_func__call__tupler   )r>   r   r   r   r   r   r   r   r   r   r   all_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cacheilayer_modulelayer_head_maskr   layer_outputss                       rA   rY   zLayoutLMEncoder.forward  s    #7BD$5b4%64;;;Z;Zr`d&&4==##p "	#,R$(4 #	VOA|#$58H$H!.7.CilO3B3N_Q/TXN**t}} $ A A ))!"#)*"%	! !-!"#)*"%! *!,M"}R'8&::" &9]1=M<O&O#;;22+?=QRCSBU+U(G#	VJ   1]4D D 
 "&%'(
 
 
 9+.+*1
 	
rB   )	NNNNNNFFT)rZ   r[   r\   r$   r:   r   r   r   r   r   r   r   rY   r^   r_   s   @rA   r   r     s   , 7;15=A>BEI$(,1/4&*S
||S
 !!2!23S
 E--.	S

  ((9(9:S
 !)):): ;S
 "%e.?.?(@"ABS
 D>S
 $D>S
 'tnS
 d^S
 
uU\\"$MM	NS
rB   r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )LayoutLMPoolerc                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )r#   r$   r   ro   r'   r   Tanh
activationr=   s     rA   r$   zLayoutLMPooler.__init__  s9    YYv1163E3EF
'')rB   r   ry   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r  )r>   r   first_token_tensorpooled_outputs       rA   rY   zLayoutLMPooler.forward$  s6     +1a40

#566rB   r   r_   s   @rA   r  r    s#    $
U\\ ell rB   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )LayoutLMPredictionHeadTransformc                 h   t         |           t        j                  |j                  |j                        | _        t        |j                  t              rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                        | _        y r   )r#   r$   r   ro   r'   r   r   r   r   r   transform_act_fnr5   r4   r=   s     rA   r$   z(LayoutLMPredictionHeadTransform.__init__/  s{    YYv1163E3EF
f''-$*6+<+<$=D!$*$5$5D!f&8&8f>S>STrB   r   ry   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r5   r   s     rA   rY   z'LayoutLMPredictionHeadTransform.forward8  s4    

=1--m<}5rB   r   r_   s   @rA   r  r  .  s$    UU\\ ell rB   r  c                   *     e Zd Z fdZd Zd Z xZS )LayoutLMLMPredictionHeadc                 H   t         |           t        |      | _        t	        j
                  |j                  |j                  d      | _        t	        j                  t        j                  |j                              | _        | j                  | j                  _        y )NF)bias)r#   r$   r  	transformr   ro   r'   r&   decoder	Parameterr:   rI   r%  r=   s     rA   r$   z!LayoutLMLMPredictionHead.__init__A  sm    8@ yy!3!3V5F5FUSLLV->->!?@	 !IIrB   c                 :    | j                   | j                  _         y r   )r%  r'  r>   s    rA   _tie_weightsz%LayoutLMLMPredictionHead._tie_weightsN  s     IIrB   c                 J    | j                  |      }| j                  |      }|S r   )r&  r'  r   s     rA   rY   z LayoutLMLMPredictionHead.forwardQ  s$    }5]3rB   )rZ   r[   r\   r$   r+  rY   r^   r_   s   @rA   r#  r#  @  s    &&rB   r#  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )LayoutLMOnlyMLMHeadc                 B    t         |           t        |      | _        y r   )r#   r$   r#  predictionsr=   s     rA   r$   zLayoutLMOnlyMLMHead.__init__Y  s    3F;rB   sequence_outputry   c                 (    | j                  |      }|S r   )r0  )r>   r1  prediction_scoress      rA   rY   zLayoutLMOnlyMLMHead.forward]  s     ,,_=  rB   r   r_   s   @rA   r.  r.  X  s#    <!u|| ! !rB   r.  c                       e Zd ZeZdZdZd Zy)LayoutLMPreTrainedModellayoutlmTc                 X   t        |t        j                        rm|j                  j                  j                  d| j                  j                         |j                  %|j                  j                  j                          yyt        |t        j                        rz|j                  j                  j                  d| j                  j                         |j                  2|j                  j                  |j                     j                          yyt        |t              rJ|j                  j                  j                          |j                  j                  j                  d       yt        |t              r%|j                  j                  j                          yy)zInitialize the weightsg        )meanstdN      ?)r   r   ro   weightdatanormal_r?   initializer_ranger%  zero_r%   r   r3   fill_r#  )r>   modules     rA   _init_weightsz%LayoutLMPreTrainedModel._init_weightsh  s$   fbii( MM&&CT[[5R5R&S{{&  &&( '-MM&&CT[[5R5R&S!!-""6#5#56<<> . 12KK""$MM$$S) 89KK""$ :rB   N)rZ   r[   r\   r   config_classbase_model_prefixsupports_gradient_checkpointingrB  r   rB   rA   r5  r5  b  s    !L"&*#%rB   r5  c                       e Zd Z fdZd Zd Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee	j                     d	ee	j                     d
ee	j                     dee	j                     dee	j                     dee	j                     dee   dee   dee   deeef   fd       Z xZS )LayoutLMModelc                     t         t        |   |       || _        t	        |      | _        t        |      | _        t        |      | _	        | j                          y r   )r#   rG  r$   r?   r   rX   r   encoderr  pooler	post_initr=   s     rA   r$   zLayoutLMModel.__init__}  sI    mT+F3,V4&v.$V, 	rB   c                 .    | j                   j                  S r   rX   r)   r*  s    rA   get_input_embeddingsz"LayoutLMModel.get_input_embeddings  s    ...rB   c                 &    || j                   _        y r   rM  )r>   rr   s     rA   set_input_embeddingsz"LayoutLMModel.set_input_embeddings  s    */'rB   c                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsrI  r   r   r   )r>   heads_to_pruner   r   s       rA   _prune_headszLayoutLMModel._prune_heads  sE    
 +002 	CLE5LLu%//;;EB	CrB   rL   rM   r   rN   r    r   rO   r   r   r   r   r   ry   c                 >   |
|
n| j                   j                  }
||n| j                   j                  }||n| j                   j                  }||t	        d      |#| j                  ||       |j                         }n!||j                         dd }nt	        d      ||j                  n|j                  }|t        j                  ||      }|&t        j                  |t        j                  |      }|)t        j                  |dz   t        j                  |      }|j                  d      j                  d	      }|j                  | j                  
      }d|z
  t        j                  | j                        j                   z  }||j#                         dk(  rh|j                  d      j                  d      j                  d      j                  d      }|j%                  | j                   j&                  dddd      }nB|j#                         d	k(  r/|j                  d      j                  d      j                  d      }|j                  t)        | j+                               j                  
      }ndg| j                   j&                  z  }| j-                  |||||      }| j/                  ||||
||      }|d   }| j1                  |      }|s
||f|dd z   S t3        |||j4                  |j6                  |j8                        S )a  
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMModel
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "world"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])

        >>> outputs = model(
        ...     input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids
        ... )

        >>> last_hidden_states = outputs.last_hidden_state
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer!   z5You have to specify either input_ids or inputs_embeds)rF   rD   )   r   rG   r   r:  r   )rL   rM   r    rN   rO   )r   r   r   r   )r  pooler_outputr   r  r  )r?   r   r   use_return_dictrk   %warn_if_padding_and_no_attention_maskrH   rF   r:   onesrI   rJ   	unsqueezer   rE   finfominr   r<   r   next
parametersrX   rI  rJ  r   r   r  r  )r>   rL   rM   r   rN   r    r   rO   r   r   r   r   r   rP   rF   extended_attention_maskembedding_outputencoder_outputsr1  r  s                       rA   rY   zLayoutLMModel.forward  s   l 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B] ]%>cdd"66y.Q#..*K&',,.s3KTUU%.%:!!@T@T!"ZZFCN!"[[EJJvVN<;;{T1FSD"0":":1"="G"G"J"9"<"<4::"<"N#&)@#@EKKPTPZPZD[D_D_"_ }}!#%//2<<Q?II"MWWXZ[	%,,T[[-J-JBPRTVXZ[	A%%//2<<R@JJ2N	!40A+B+H+HII!>!>>I??%)' + 
 ,,#/!5# ' 
 *!,O4#]3oab6III;-')77&11,==
 	
rB   NNNNNNNNNNNN)rZ   r[   r\   r$   rN  rP  rT  r   r   r:   
LongTensorr   r   r   r   r   rY   r^   r_   s   @rA   rG  rG  {  s[   	/0C  15+/6:59371559=A>B,0/3&*y
E,,-y
 u''(y
 !!2!23	y

 !!1!12y
 u//0y
 E--.y
   1 12y
  ((9(9:y
 !)):): ;y
 $D>y
 'tny
 d^y
 
uBB	Cy
 y
rB   rG  c                        e Zd ZddgZ fdZd Zd Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     d	e	e
j                     d
e	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e   de	e   de	e   deeef   fd       Z xZS )LayoutLMForMaskedLMzcls.predictions.decoder.biaszcls.predictions.decoder.weightc                     t         |   |       t        |      | _        t	        |      | _        | j                          y r   )r#   r$   rG  r6  r.  clsrK  r=   s     rA   r$   zLayoutLMForMaskedLM.__init__  s4     %f-&v. 	rB   c                 B    | j                   j                  j                  S r   r6  rX   r)   r*  s    rA   rN  z(LayoutLMForMaskedLM.get_input_embeddings       }}''777rB   c                 B    | j                   j                  j                  S r   )rh  r0  r'  r*  s    rA   get_output_embeddingsz)LayoutLMForMaskedLM.get_output_embeddings#  s    xx##+++rB   c                     || j                   j                  _        |j                  | j                   j                  _        y r   )rh  r0  r'  r%  )r>   new_embeddingss     rA   set_output_embeddingsz)LayoutLMForMaskedLM.set_output_embeddings&  s,    '5$$2$7$7!rB   rL   rM   r   rN   r    r   rO   labelsr   r   r   r   r   ry   c                    ||n| j                   j                  }| j                  ||||||||	|
|||      }|d   }| j                  |      }d}|Ft	               } ||j                  d| j                   j                        |j                  d            }|s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )a2	  
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForMaskedLM
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMForMaskedLM.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "[MASK]"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])

        >>> labels = tokenizer("Hello world", return_tensors="pt")["input_ids"]

        >>> outputs = model(
        ...     input_ids=input_ids,
        ...     bbox=bbox,
        ...     attention_mask=attention_mask,
        ...     token_type_ids=token_type_ids,
        ...     labels=labels,
        ... )

        >>> loss = outputs.loss
        ```N)
r   rN   r    r   rO   r   r   r   r   r   r   r!   rG   losslogitsr   r  )
r?   rX  r6  rh  r   r{   r&   r   r   r  )r>   rL   rM   r   rN   r    r   rO   rq  r   r   r   r   r   r   r1  r3  masked_lm_lossloss_fctr   s                       rA   rY   zLayoutLMForMaskedLM.forward*  s   B &1%<k$++B]B]--))%'"7#9/!5#   
 "!* HH_5')H%!&&r4;;+A+ABBN
 ')GABK7F3A3M^%.YSYY$!//))	
 	
rB   )NNNNNNNNNNNNN)rZ   r[   r\   _tied_weights_keysr$   rN  rm  rp  r   r   r:   rd  r   r   r   r   r   rY   r^   r_   s   @rA   rf  rf    s~   8:Z[8,8  15+/6:59371559-1=A>B,0/3&*e
E,,-e
 u''(e
 !!2!23	e

 !!1!12e
 u//0e
 E--.e
   1 12e
 ))*e
  ((9(9:e
 !)):): ;e
 $D>e
 'tne
 d^e
 
un$	%e
 e
rB   rf  z
    LayoutLM Model with a sequence classification head on top (a linear layer on top of the pooled output) e.g. for
    document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset.
    )custom_introc                   j    e Zd Z fdZd Ze	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee
   dee
   dee
   deeef   fd       Z xZS )!LayoutLMForSequenceClassificationc                 ,   t         |   |       |j                  | _        t        |      | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r   r#   r$   
num_labelsrG  r6  r   r6   r7   r8   ro   r'   
classifierrK  r=   s     rA   r$   z*LayoutLMForSequenceClassification.__init__  i      ++%f-zz&"<"<=))F$6$68I8IJ 	rB   c                 B    | j                   j                  j                  S r   rj  r*  s    rA   rN  z6LayoutLMForSequenceClassification.get_input_embeddings  rk  rB   rL   rM   r   rN   r    r   rO   rq  r   r   r   ry   c                 B   ||n| j                   j                  }| j                  ||||||||	|
|
      }|d   }| j                  |      }| j	                  |      }d}|| j                   j
                  | j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j
                  dk(  rIt               }| j                  dk(  r& ||j                         |j                               }n |||      }n| j                   j
                  dk(  r=t               } ||j                  d| j                        |j                  d            }n,| j                   j
                  dk(  rt               } |||      }|s|f|dd z   }||f|z   S |S t!        |||j"                  |j$                  	      S )
aB	  
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForSequenceClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "world"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])
        >>> sequence_label = torch.tensor([1])

        >>> outputs = model(
        ...     input_ids=input_ids,
        ...     bbox=bbox,
        ...     attention_mask=attention_mask,
        ...     token_type_ids=token_type_ids,
        ...     labels=sequence_label,
        ... )

        >>> loss = outputs.loss
        >>> logits = outputs.logits
        ```N
rL   rM   r   rN   r    r   rO   r   r   r   r   
regressionsingle_label_classificationmulti_label_classificationr!   rG   rs  )r?   rX  r6  r8   r  problem_typer~  rE   r:   rJ   rl   r	   squeezer   r{   r   r   r   r  )r>   rL   rM   r   rN   r    r   rO   rq  r   r   r   r   r  ru  rt  rw  r   s                     rA   rY   z)LayoutLMForSequenceClassification.forward  s   ~ &1%<k$++B]B]--))%'/!5#   
  
]3/{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE'!//))	
 	
rB   NNNNNNNNNNN)rZ   r[   r\   r$   rN  r   r   r:   rd  r   r   r   r   r   rY   r^   r_   s   @rA   r{  r{    s7   8  15+/6:59371559-1,0/3&*q
E,,-q
 u''(q
 !!2!23	q

 !!1!12q
 u//0q
 E--.q
   1 12q
 ))*q
 $D>q
 'tnq
 d^q
 
u..	/q
 q
rB   r{  a3  
    LayoutLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    sequence labeling (information extraction) tasks such as the [FUNSD](https://guillaumejaume.github.io/FUNSD/)
    dataset and the [SROIE](https://rrc.cvc.uab.es/?ch=13) dataset.
    c                   j    e Zd Z fdZd Ze	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee
   dee
   dee
   deeef   fd       Z xZS )LayoutLMForTokenClassificationc                 ,   t         |   |       |j                  | _        t        |      | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r   r}  r=   s     rA   r$   z'LayoutLMForTokenClassification.__init__$  r  rB   c                 B    | j                   j                  j                  S r   rj  r*  s    rA   rN  z3LayoutLMForTokenClassification.get_input_embeddings.  rk  rB   rL   rM   r   rN   r    r   rO   rq  r   r   r   ry   c                    ||n| j                   j                  }| j                  ||||||||	|
|
      }|d   }| j                  |      }| j	                  |      }d}|<t               } ||j                  d| j                        |j                  d            }|s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )a  
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForTokenClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased")
        >>> model = LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased")

        >>> words = ["Hello", "world"]
        >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782]

        >>> token_boxes = []
        >>> for word, box in zip(words, normalized_word_boxes):
        ...     word_tokens = tokenizer.tokenize(word)
        ...     token_boxes.extend([box] * len(word_tokens))
        >>> # add bounding boxes of cls + sep tokens
        >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]]

        >>> encoding = tokenizer(" ".join(words), return_tensors="pt")
        >>> input_ids = encoding["input_ids"]
        >>> attention_mask = encoding["attention_mask"]
        >>> token_type_ids = encoding["token_type_ids"]
        >>> bbox = torch.tensor([token_boxes])
        >>> token_labels = torch.tensor([1, 1, 0, 0]).unsqueeze(0)  # batch size of 1

        >>> outputs = model(
        ...     input_ids=input_ids,
        ...     bbox=bbox,
        ...     attention_mask=attention_mask,
        ...     token_type_ids=token_type_ids,
        ...     labels=token_labels,
        ... )

        >>> loss = outputs.loss
        >>> logits = outputs.logits
        ```Nr  r   r!   rG   rs  )r?   rX  r6  r8   r  r   r{   r~  r   r   r  )r>   rL   rM   r   rN   r    r   rO   rq  r   r   r   r   r1  ru  rt  rw  r   s                     rA   rY   z&LayoutLMForTokenClassification.forward1  s    z &1%<k$++B]B]--))%'/!5#   
 "!*,,71')HFKKDOO<fkk"oNDY,F)-)9TGf$EvE$!//))	
 	
rB   r  )rZ   r[   r\   r$   rN  r   r   r:   rd  r   r   r   r   r   rY   r^   r_   s   @rA   r  r    s7   8  15+/6:59371559-1,0/3&*^
E,,-^
 u''(^
 !!2!23	^

 !!1!12^
 u//0^
 E--.^
   1 12^
 ))*^
 $D>^
 'tn^
 d^^
 
u++	,^
 ^
rB   r  c                       e Zd Zd fd	Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     deej                     dee
   dee
   dee
   deeef   fd       Z xZS )LayoutLMForQuestionAnsweringc                     t         |   |       |j                  | _        t        |      | _        t        j                  |j                  |j                        | _        | j                          y)z
        has_visual_segment_embedding (`bool`, *optional*, defaults to `True`):
            Whether or not to add visual segment embeddings.
        N)
r#   r$   r~  rG  r6  r   ro   r'   
qa_outputsrK  )r>   r?   has_visual_segment_embeddingr@   s      rA   r$   z%LayoutLMForQuestionAnswering.__init__  sU    
 	  ++%f-))F$6$68I8IJ 	rB   c                 B    | j                   j                  j                  S r   rj  r*  s    rA   rN  z1LayoutLMForQuestionAnswering.get_input_embeddings  rk  rB   rL   rM   r   rN   r    r   rO   start_positionsend_positionsr   r   r   ry   c                 *   ||n| j                   j                  }| j                  ||||||||
||
      }|d   }| j                  |      }|j	                  dd      \  }}|j                  d      j                         }|j                  d      j                         }d}||	t        |j                               dkD  r|j                  d      }t        |	j                               dkD  r|	j                  d      }	|j                  d      }|j                  d|      }|	j                  d|      }	t        |      } |||      } |||	      }||z   dz  }|s||f|dd z   }||f|z   S |S t        ||||j                  |j                  	      S )
aL	  
        bbox (`torch.LongTensor` of shape `(batch_size, sequence_length, 4)`, *optional*):
            Bounding boxes of each input sequence tokens. Selected in the range `[0,
            config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
            format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
            y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.

        Example:

        In the example below, we prepare a question + context pair for the LayoutLM model. It will give us a prediction
        of what it thinks the answer is (the span of the answer within the texts parsed from the image).

        ```python
        >>> from transformers import AutoTokenizer, LayoutLMForQuestionAnswering
        >>> from datasets import load_dataset
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("impira/layoutlm-document-qa", add_prefix_space=True)
        >>> model = LayoutLMForQuestionAnswering.from_pretrained("impira/layoutlm-document-qa", revision="1e3ebac")

        >>> dataset = load_dataset("nielsr/funsd", split="train", trust_remote_code=True)
        >>> example = dataset[0]
        >>> question = "what's his name?"
        >>> words = example["words"]
        >>> boxes = example["bboxes"]

        >>> encoding = tokenizer(
        ...     question.split(), words, is_split_into_words=True, return_token_type_ids=True, return_tensors="pt"
        ... )
        >>> bbox = []
        >>> for i, s, w in zip(encoding.input_ids[0], encoding.sequence_ids(0), encoding.word_ids(0)):
        ...     if s == 1:
        ...         bbox.append(boxes[w])
        ...     elif i == tokenizer.sep_token_id:
        ...         bbox.append([1000] * 4)
        ...     else:
        ...         bbox.append([0] * 4)
        >>> encoding["bbox"] = torch.tensor([bbox])

        >>> word_ids = encoding.word_ids(0)
        >>> outputs = model(**encoding)
        >>> loss = outputs.loss
        >>> start_scores = outputs.start_logits
        >>> end_scores = outputs.end_logits
        >>> start, end = word_ids[start_scores.argmax(-1)], word_ids[end_scores.argmax(-1)]
        >>> print(" ".join(words[start : end + 1]))
        M. Hamann P. Harper, P. Martinez
        ```Nr  r   r   r!   r   )ignore_indexrG   )rt  start_logits
end_logitsr   r  )r?   rX  r6  r  splitr  r   r   rH   clampr   r   r   r  )r>   rL   rM   r   rN   r    r   rO   r  r  r   r   r   r   r1  ru  r  r  
total_lossignored_indexrw  
start_lossend_lossr   s                           rA   rY   z$LayoutLMForQuestionAnswering.forward  s   B &1%<k$++B]B]--))%'/!5#   
 "!*1#)<<r<#: j#++B/::<''+668

&=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J"J/'!"+=F/9/EZMF*Q6Q+%!!//))
 	
rB   )Trc  )rZ   r[   r\   r$   rN  r   r   r:   rd  r   r   r   r   r   rY   r^   r_   s   @rA   r  r    sP   8  15+/6:593715596:48,0/3&*q
E,,-q
 u''(q
 !!2!23	q

 !!1!12q
 u//0q
 E--.q
   1 12q
 "%"2"23q
   0 01q
 $D>q
 'tnq
 d^q
 
u22	3q
 q
rB   r  )rf  r{  r  r  rG  r5  );r]   r   typingr   r   r   r:   torch.utils.checkpointr   torch.nnr   r   r	   activationsr   modeling_outputsr   r   r   r   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   configuration_layoutlmr   
get_loggerrZ   r  r5   r3   Moduler   ra   r   r   r   r   r   r   r   r  r  r#  r.  r5  rG  rf  r{  r  r  __all__r   rB   rA   <module>r     s     ) )    A A !  . l l , 2 
		H	% LL I IZCBII CN  "# 0		 0h299  RYY SBII SnZ
bii Z
|RYY  bii $ryy 0!")) ! %o % %0 T
+ T
 T
n |
1 |
 |
~ @
(? @
@
F m
%< m
m
` D
#: D
 D
NrB   