
    Uh7                        d Z ddlZddlZddlmZmZmZmZm	Z	m
Z
mZ ddlZddlZddlmZ ddlmZmZmZ ddlmZ ddlmZmZmZmZ dd	lmZmZ dd
lmZmZ ddl m!Z!m"Z"m#Z# ddl$m%Z%  e"jL                  e'      Z( G d dejR                        Z* G d dejR                        Z+	 d6dejR                  dejX                  dejX                  dejX                  deejX                     de-de-fdZ. G d dejR                        Z/ G d dejR                        Z0 G d dejR                        Z1 G d  d!ejR                        Z2 G d" d#ejR                        Z3 G d$ d%ejR                        Z4 G d& d'ejR                        Z5e! G d( d)e             Z6e! G d* d+e6             Z7 G d, d-ejR                        Z8 e!d./       G d0 d1e6             Z9 e!d2/       G d3 d4e6             Z:g d5Z;y)7zPyTorch ViT model.    N)CallableDictListOptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedImageModelingOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)auto_docstringlogging	torch_int   )	ViTConfigc            	            e Zd ZdZddededdf fdZdej                  de	d	e	dej                  fd
Z
	 	 ddej                  deej                     dedej                  fdZ xZS )ViTEmbeddingszb
    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
    configuse_mask_tokenreturnNc                 J   t         |           t        j                  t	        j
                  dd|j                              | _        |r4t        j                  t	        j                  dd|j                              nd | _	        t        |      | _        | j                  j                  }t        j                  t	        j
                  d|dz   |j                              | _        t        j                  |j                        | _        |j"                  | _        || _        y )Nr   )super__init__r
   	Parametertorchrandnhidden_size	cls_tokenzeros
mask_tokenViTPatchEmbeddingspatch_embeddingsnum_patchesposition_embeddingsDropouthidden_dropout_probdropout
patch_sizer   )selfr   r    r.   	__class__s       v/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/vit/modeling_vit.pyr$   zViTEmbeddings.__init__/   s    ekk!Q8J8J&KLQ_",,u{{1a9K9K'LMei 26 :++77#%<<A{QPVPbPb0c#d zz&"<"<= ++    
embeddingsheightwidthc                    |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   N      ?r   r      bicubicF)sizemodealign_cornersdim)shaper/   r&   jit
is_tracingr3   r   reshapepermuter
   
functionalinterpolateviewcat)r4   r8   r9   r:   r.   num_positionsclass_pos_embedpatch_pos_embedrD   
new_height	new_widthsqrt_num_positionss               r6   interpolate_pos_encodingz&ViTEmbeddings.interpolate_pos_encoding;   s`    !&&q)A-0066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"t.
T__,	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr7   pixel_valuesbool_masked_posrT   c                    |j                   \  }}}}| j                  ||      }|Z|j                   d   }	| j                  j                  ||	d      }
|j	                  d      j                  |
      }|d|z
  z  |
|z  z   }| j                  j                  |dd      }t        j                  ||fd      }|r|| j                  |||      z   }n|| j                  z   }| j                  |      }|S )N)rT   r   r<         ?rC   )rE   r-   r+   expand	unsqueezetype_asr)   r&   rM   rT   r/   r2   )r4   rU   rV   rT   
batch_sizenum_channelsr9   r:   r8   
seq_lengthmask_tokensmask
cls_tokenss                r6   forwardzViTEmbeddings.forwardc   s    3?2D2D/
L&%**<Rj*k
&#))!,J//00ZLK",,R088ED#sTz2[45GGJ ^^**:r2>
YY
J7Q?
 $#d&C&CJPVX]&^^J#d&>&>>J\\*-
r7   FNF)__name__
__module____qualname____doc__r   boolr$   r&   TensorintrT   r   
BoolTensorrb   __classcell__r5   s   @r6   r   r   *   s    
y 
$ 
4 
&D5<< &D &DUX &D]b]i]i &DV 7;).	ll "%"2"23 #'	
 
r7   r   c                   `     e Zd ZdZ fdZddej                  dedej                  fdZ xZ	S )r,   z
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                    t         |           |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                        r|n||f}t        |t        j                  j                        r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||      | _        y )Nr   r   )kernel_sizestride)r#   r$   
image_sizer3   r]   r(   
isinstancecollectionsabcIterabler.   r
   Conv2d
projection)r4   r   rs   r3   r]   r(   r.   r5   s          r6   r$   zViTPatchEmbeddings.__init__   s    !'!2!2F4E4EJ
$*$7$79K9Kk#-j+//:R:R#SZZdfpYq
#-j+//:R:R#SZZdfpYq
!!}
15*Q-:VW=:XY$$(&))L+:^hir7   rU   rT   r!   c                    |j                   \  }}}}|| j                  k7  rt        d| j                   d| d      |sV|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d		      | j	                  |      j                  d
      j                  dd
      }|S )NzoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got .r   r   zInput image size (*z) doesn't match model (z).r>   )rE   r]   
ValueErrorrs   ry   flatten	transpose)r4   rU   rT   r\   r]   r9   r:   r8   s           r6   rb   zViTPatchEmbeddings.forward   s    2>2D2D/
L&%4,,,!../yaI  (++u8J/J (% 9+,Adooa.@-AE  __\2::1=GG1M
r7   rc   )
re   rf   rg   rh   r$   r&   rj   ri   rb   rm   rn   s   @r6   r,   r,      s3    jELL D ]b]i]i r7   r,   modulequerykeyvalueattention_maskscalingr2   c                    t        j                  ||j                  dd            |z  }t        j                  j                  |dt         j                        j                  |j                        }t        j                  j                  ||| j                        }|||z  }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr<   )rD   dtype)ptrainingr   r>   )r&   matmulr   r
   rJ   softmaxfloat32tor   r2   r   
contiguous)
r   r   r   r   r   r   r2   kwargsattn_weightsattn_outputs
             r6   eager_attention_forwardr      s     <<s}}R'<=GL ==((2U]](SVVW\WbWbcL ==((6??([L !#n4,,|U3K''1-88:K$$r7   c            
            e Zd Zdeddf fdZdej                  dej                  fdZ	 d
deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )ViTSelfAttentionr   r!   Nc                 2   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        |j                  | _        | j                  dz  | _        d| _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        t        j                  |j                  | j                  |j                         | _        y )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r{   g      F)bias)r#   r$   r(   num_attention_headshasattrr}   r   rk   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probr   	is_causalr
   Linearqkv_biasr   r   r   r4   r   r5   s     r6   r$   zViTSelfAttention.__init__   sF    : ::a?PVXhHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r7   xc                     |j                         d d | j                  | j                  fz   }|j                  |      }|j	                  dddd      S )Nr<   r   r>   r   r   )r@   r   r   rL   rI   )r4   r   new_x_shapes      r6   transpose_for_scoresz%ViTSelfAttention.transpose_for_scores   sL    ffhsmt'?'?AYAY&ZZFF;yyAq!$$r7   	head_maskoutput_attentionsc           
         | j                  | j                  |            }| j                  | j                  |            }| j                  | j                  |            }t        }| j
                  j                  dk7  rN| j
                  j                  dk(  r|rt        j                  d       nt        | j
                  j                     } || ||||| j                  | j                  | j                  sdn| j                        \  }}	|j                         d d | j                  fz   }
|j!                  |
      }|r||	f}|S |f}|S )Neagersdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.        )r   r   r2   r   )r   r   r   r   r   r   _attn_implementationloggerwarning_oncer   r   r   r   r   r@   r   rH   )r4   hidden_statesr   r   	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapeoutputss               r6   rb   zViTSelfAttention.forward   s=    --dhh}.EF	//

=0IJ//

=0IJ(?;;++w6{{//69>O##L
 '>dkk>^>^&_#)<nnLL#}}C$2C2C	*
& #0"4"4"6s";t?Q?Q>S"S%--.EF6G=/2 O\M]r7   rd   )re   rf   rg   r   r$   r&   rj   r   r   ri   r	   r   rb   rm   rn   s   @r6   r   r      s    ]y ]T ](%ell %u|| % bg!(0(>!Z^!	uU\\5<</0%2EE	F!r7   r   c                   |     e Zd ZdZdeddf fdZdej                  dej                  dej                  fdZ xZ	S )	ViTSelfOutputz
    The residual connection is defined in ViTLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    r   r!   Nc                     t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        y N)	r#   r$   r
   r   r(   denser0   r1   r2   r   s     r6   r$   zViTSelfOutput.__init__  sB    YYv1163E3EF
zz&"<"<=r7   r   input_tensorc                 J    | j                  |      }| j                  |      }|S r   r   r2   r4   r   r   s      r6   rb   zViTSelfOutput.forward  s$    

=1]3r7   )
re   rf   rg   rh   r   r$   r&   rj   rb   rm   rn   s   @r6   r   r     sD    
>y >T >
U\\  RWR^R^ r7   r   c                        e Zd Zdeddf fdZdee   ddfdZ	 	 ddej                  de
ej                     d	edeeej                  ej                  f   eej                     f   fd
Z xZS )ViTAttentionr   r!   Nc                     t         |           t        |      | _        t	        |      | _        t               | _        y r   )r#   r$   r   	attentionr   outputsetpruned_headsr   s     r6   r$   zViTAttention.__init__  s0    )&1#F+Er7   headsc                 >   t        |      dk(  ry t        || j                  j                  | j                  j                  | j
                        \  }}t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _        t        | j                  j                  |      | j                  _	        t        | j                  j                  |d      | j                  _        | j                  j                  t        |      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |      | _        y )Nr   r   rC   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r4   r   indexs      r6   prune_headszViTAttention.prune_heads  s   u:?74>>55t~~7Y7Y[_[l[l
u
  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r7   r   r   r   c                 h    | j                  |||      }| j                  |d   |      }|f|dd  z   }|S )Nr   r   )r   r   )r4   r   r   r   self_outputsattention_outputr   s          r6   rb   zViTAttention.forward0  sE     ~~mY@QR;;|AF#%QR(88r7   rd   )re   rf   rg   r   r$   r   rk   r   r&   rj   r   ri   r	   r   rb   rm   rn   s   @r6   r   r     s    "y "T ";S ;d ;* -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr7   r   c                   `     e Zd Zdeddf fdZdej                  dej                  fdZ xZS )ViTIntermediater   r!   Nc                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r#   r$   r
   r   r(   intermediate_sizer   rt   
hidden_actstrr   intermediate_act_fnr   s     r6   r$   zViTIntermediate.__init__?  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r7   r   c                 J    | j                  |      }| j                  |      }|S r   )r   r   )r4   r   s     r6   rb   zViTIntermediate.forwardG  s&    

=100?r7   	re   rf   rg   r   r$   r&   rj   rb   rm   rn   s   @r6   r   r   >  s1    9y 9T 9U\\ ell r7   r   c                   x     e Zd Zdeddf fdZdej                  dej                  dej                  fdZ xZS )	ViTOutputr   r!   Nc                     t         |           t        j                  |j                  |j
                        | _        t        j                  |j                        | _	        y r   )
r#   r$   r
   r   r   r(   r   r0   r1   r2   r   s     r6   r$   zViTOutput.__init__O  sB    YYv779K9KL
zz&"<"<=r7   r   r   c                 T    | j                  |      }| j                  |      }||z   }|S r   r   r   s      r6   rb   zViTOutput.forwardT  s.    

=1]3%4r7   r   rn   s   @r6   r   r   N  s?    >y >T >
U\\  RWR^R^ r7   r   c                        e Zd ZdZdeddf fdZ	 	 d
dej                  deej                     de	de
eej                  ej                  f   eej                     f   fd	Z xZS )ViTLayerz?This corresponds to the Block class in the timm implementation.r   r!   Nc                 r   t         |           |j                  | _        d| _        t	        |      | _        t        |      | _        t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   eps)r#   r$   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r   r
   	LayerNormr(   layer_norm_epslayernorm_beforelayernorm_afterr   s     r6   r$   zViTLayer.__init__`  s    '-'E'E$%f-+F3' "V-?-?VEZEZ [!||F,>,>FDYDYZr7   r   r   r   c                     | j                  | j                  |      ||      }|d   }|dd  }||z   }| j                  |      }| j                  |      }| j	                  ||      }|f|z   }|S )N)r   r   r   )r   r   r   r   r   )r4   r   r   r   self_attention_outputsr   r   layer_outputs           r6   rb   zViTLayer.forwardj  s     "&!!-0/ "0 "

 2!4(, )=8 ++M:((6 {{<?/G+r7   rd   )re   rf   rg   rh   r   r$   r&   rj   r   ri   r	   r   rb   rm   rn   s   @r6   r   r   ]  s    I[y [T [ -1"'	|| ELL)  	
 
uU\\5<</0%2EE	Fr7   r   c                        e Zd Zdeddf fdZ	 	 	 	 ddej                  deej                     deded	ede	e
ef   fd
Z xZS )
ViTEncoderr   r!   Nc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w rd   )
r#   r$   r   r
   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r4   r   _r5   s      r6   r$   zViTEncoder.__init__  sN    ]]eFD\D\>]#^HV$4#^_
&+# $_s   A#r   r   r   output_hidden_statesreturn_dictc                 t   |rdnd }|rdnd }t        | j                        D ]h  \  }}	|r||fz   }|||   nd }
| j                  r+| j                  r| j	                  |	j
                  ||
|      }n
 |	||
|      }|d   }|s`||d   fz   }j |r||fz   }|st        d |||fD              S t        |||      S )N r   r   c              3   &   K   | ]	  }||  y wr   r   ).0vs     r6   	<genexpr>z%ViTEncoder.forward.<locals>.<genexpr>  s     mq_`_lms   )last_hidden_stater   
attentions)	enumerater   r   r   _gradient_checkpointing_func__call__tupler   )r4   r   r   r   r   r   all_hidden_statesall_self_attentionsilayer_modulelayer_head_masklayer_outputss               r6   rb   zViTEncoder.forward  s     #7BD$5b4(4 	POA|#$58H$H!.7.CilO**t}} $ A A ))!#%	! !-]OM^ _)!,M &9]1=M<O&O#'	P*   1]4D Dm]4EGZ$[mmm++*
 	
r7   )NFFT)re   rf   rg   r   r$   r&   rj   r   ri   r	   r  r   rb   rm   rn   s   @r6   r   r     sz    ,y ,T , -1"'%* )
||)
 ELL))
  	)

 #)
 )
 
uo%	&)
r7   r   c                       e Zd ZeZdZdZdZddgZdZ	dZ
deej                  ej                  ej                  f   ddfd	Zy)
ViTPreTrainedModelvitrU   Tr   r   r   r!   Nc                    t        |t        j                  t        j                  f      rt        j                  j                  |j                  j                  j                  t        j                        d| j                  j                        j                  |j                  j                        |j                  _        |j                  %|j                  j                  j                          yyt        |t        j                         rJ|j                  j                  j                          |j                  j                  j#                  d       yt        |t$              rdt        j                  j                  |j&                  j                  j                  t        j                        d| j                  j                        j                  |j&                  j                        |j&                  _        t        j                  j                  |j(                  j                  j                  t        j                        d| j                  j                        j                  |j(                  j                        |j(                  _        |j*                  %|j*                  j                  j                          yyy)zInitialize the weightsr   )meanstdNrX   )rt   r
   r   rx   inittrunc_normal_weightdatar   r&   r   r   initializer_ranger   r   zero_r   fill_r   r/   r)   r+   )r4   r   s     r6   _init_weightsz ViTPreTrainedModel._init_weights  s   fryy"))45 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '-KK""$MM$$S)..0gg.C.C**//225==AKK11 /D / b++112	 &&+ %'GG$9$9  %%((7KK11 %: % b!!''(	 !   ,!!&&,,. - /r7   )re   rf   rg   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_2r	   r
   r   rx   r   r  r   r7   r6   r  r    sZ    L$O&*#(*5N!/E"))RYY*L$M /RV /r7   r  c                       e Zd Zddededef fdZdefdZdee	e
e	   f   ddfd	Ze	 	 	 	 	 	 	 dd
eej                     deej                      deej                     dee   dee   dee   dee   deeef   fd       Z xZS )ViTModelr   add_pooling_layerr    c                    t         |   |       || _        t        ||      | _        t        |      | _        t        j                  |j                  |j                        | _        |rt        |      nd| _        | j                          y)z
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        use_mask_token (`bool`, *optional*, defaults to `False`):
            Whether to use a mask token for masked image modeling.
        )r    r   N)r#   r$   r   r   r8   r   encoderr
   r   r(   r   	layernorm	ViTPoolerpooler	post_init)r4   r   r"  r    r5   s       r6   r$   zViTModel.__init__  sm     	 '~N!&)f&8&8f>S>ST+<i'$ 	r7   r!   c                 .    | j                   j                  S r   )r8   r-   )r4   s    r6   get_input_embeddingszViTModel.get_input_embeddings  s    ///r7   heads_to_pruneNc                     |j                         D ]7  \  }}| j                  j                  |   j                  j	                  |       9 y)z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsr$  r   r   r   )r4   r+  r   r   s       r6   _prune_headszViTModel._prune_heads  sE    
 +002 	CLE5LLu%//;;EB	Cr7   rU   rV   r   r   r   rT   r   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }|t	        d      | j                  || j                   j                        }| j                  j                  j                  j                  j                  }|j                  |k7  r|j                  |      }| j                  |||      }	| j                  |	||||      }
|
d   }| j                  |      }| j                  | j                  |      nd}|s|||fn|f}||
dd z   S t!        |||
j"                  |
j$                        S )z
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rV   rT   )r   r   r   r   r   r   )r   pooler_outputr   r   )r   r   r   use_return_dictr}   get_head_maskr   r8   r-   ry   r  r   r   r$  r%  r'  r   r   r   )r4   rU   rV   r   r   r   rT   r   expected_dtypeembedding_outputencoder_outputssequence_outputpooled_outputhead_outputss                 r6   rb   zViTModel.forward  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]?@@ &&y$++2O2OP	 99DDKKQQ/'??>:L??/Tl + 
 ,,/!5# ' 
 *!,..98<8OO4UY?L?XO];_n^pL/!""555)-')77&11	
 	
r7   )TFNNNNNNN)re   rf   rg   r   ri   r$   r,   r*  r   rk   r   r.  r   r   r&   rj   rl   r	   r   r   rb   rm   rn   s   @r6   r!  r!    s    y T Z^ &0&8 0C4T#Y+? CD C  046:,0,0/337&*;
u||,;
 "%"2"23;
 ELL)	;

 $D>;
 'tn;
 #+4.;
 d^;
 
u00	1;
 ;
r7   r!  c                   *     e Zd Zdef fdZd Z xZS )r&  r   c                     t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        y r   )
r#   r$   r
   r   r(   pooler_output_sizer   r   
pooler_act
activationr   s     r6   r$   zViTPooler.__init__B  s>    YYv1163L3LM
 !2!23r7   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r>  )r4   r   first_token_tensorr7  s       r6   rb   zViTPooler.forwardG  s6     +1a40

#566r7   )re   rf   rg   r   r$   rb   rm   rn   s   @r6   r&  r&  A  s    4y 4
r7   r&  a[  
    ViT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://arxiv.org/abs/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )custom_introc                        e Zd Zdeddf fdZe	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee
   d	ee
   d
ee
   dee
   deeef   fd       Z xZS )ViTForMaskedImageModelingr   r!   Nc                 N   t         |   |       t        |dd      | _        t	        j
                  t	        j                  |j                  |j                  dz  |j                  z  d      t	        j                  |j                              | _        | j                          y )NFT)r"  r    r>   r   )in_channelsout_channelsrq   )r#   r$   r!  r  r
   
Sequentialrx   r(   encoder_strider]   PixelShuffledecoderr(  r   s     r6   r$   z"ViTForMaskedImageModeling.__init__]  s     FeDQ}}II"..#22A58K8KK
 OOF112
 	r7   rU   rV   r   r   r   rT   r   c           	         ||n| j                   j                  }|g| j                   j                  | j                   j                  k7  r:t	        d| j                   j                   d| j                   j                   d      | j                  |||||||      }|d   }	|	ddddf   }	|	j                  \  }
}}t        j                  |dz        x}}|	j                  dd	d      j                  |
|||      }	| j                  |	      }d}|| j                   j                  | j                   j                  z  }|j                  d
||      }|j                  | j                   j                  d      j                  | j                   j                  d	      j                  d      j                         }t         j"                  j%                  ||d      }||z  j'                         |j'                         dz   z  | j                   j(                  z  }|s|f|dd z   }||f|z   S |S t+        |||j,                  |j.                        S )a+  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, ViTForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224-in21k")
        >>> model = ViTForMaskedImageModeling.from_pretrained("google/vit-base-patch16-224-in21k")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 224, 224]
        ```NzWhen `bool_masked_pos` is provided, `patch_size` must be equal to `encoder_stride` to ensure that the reconstructed image has the same dimensions as the input. Got `patch_size` = z and `encoder_stride` = r{   )rV   r   r   r   rT   r   r   r   r=   r>   r<   none)	reductiongh㈵>)lossreconstructionr   r   )r   r1  r3   rH  r}   r  rE   mathfloorrI   rH   rJ  rs   repeat_interleaverZ   r   r
   rJ   l1_losssumr]   r   r   r   )r4   rU   rV   r   r   r   rT   r   r   r6  r\   sequence_lengthr]   r9   r:   reconstructed_pixel_valuesmasked_im_lossr@   r`   reconstruction_lossr   s                        r6   rb   z!ViTForMaskedImageModeling.forwardn  sR   L &1%<k$++B]B]&DKK,B,BdkkF`F`,`&&*kk&<&<%==UVZVaVaVpVpUqqrt  ((+/!5%=#  
 "!* *!QR%04C4I4I1
O\OS$899)11!Q:BB:|]cejk &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7F`lr"7"s1D8==?488:PTCTUX\XcXcXpXppN02WQR[@F3A3M^%.YSYY(5!//))	
 	
r7   r9  )re   rf   rg   r   r$   r   r   r&   rj   rl   ri   r	   r  r   rb   rm   rn   s   @r6   rC  rC  P  s    y T "  046:,0,0/337&*Y
u||,Y
 "%"2"23Y
 ELL)	Y

 $D>Y
 'tnY
 #+4.Y
 d^Y
 
u//	0Y
 Y
r7   rC  a  
    ViT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                        e Zd Zdeddf fdZe	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee	   d	ee	   d
ee	   dee	   de
eef   fd       Z xZS )ViTForImageClassificationr   r!   Nc                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NF)r"  r   )r#   r$   
num_labelsr!  r  r
   r   r(   Identity
classifierr(  r   s     r6   r$   z"ViTForImageClassification.__init__  ss      ++Fe< OUN_N_bcNc"))F$6$68I8IJikititiv 	r7   rU   r   labelsr   r   rT   r   c                 b   ||n| j                   j                  }| j                  ||||||      }|d   }	| j                  |	dddddf         }
d}||j	                  |
j
                        }| j                   j                  | j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt               }| j                  dk(  r& ||
j                         |j                               }n ||
|      }n| j                   j                  dk(  r=t               } ||
j                  d| j                        |j                  d            }n,| j                   j                  dk(  rt!               } ||
|      }|s|
f|dd z   }||f|z   S |S t#        ||
|j$                  |j&                  	      S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r   r   r   rT   r   r   r   
regressionsingle_label_classificationmulti_label_classificationr<   )rN  logitsr   r   )r   r1  r  r^  r   deviceproblem_typer\  r   r&   longrk   r   squeezer   rL   r   r   r   r   )r4   rU   r   r_  r   r   rT   r   r   r6  rd  rN  loss_fctr   s                 r6   rb   z!ViTForImageClassification.forward  s   " &1%<k$++B]B]((/!5%=#  
 "!*Aq!9:YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE$!//))	
 	
r7   r9  )re   rf   rg   r   r$   r   r   r&   rj   ri   r	   r  r   rb   rm   rn   s   @r6   rZ  rZ    s    
y 
T 
  04,0)-,0/337&*A
u||,A
 ELL)A
 &	A

 $D>A
 'tnA
 #+4.A
 d^A
 
u++	,A
 A
r7   rZ  )rZ  rC  r!  r  )r   )<rh   collections.abcru   rP  typingr   r   r   r   r   r   r	   r&   torch.utils.checkpointr
   torch.nnr   r   r   activationsr   modeling_outputsr   r   r   r   modeling_utilsr   r   pytorch_utilsr   r   utilsr   r   r   configuration_vitr   
get_loggerre   r   Moduler   r,   rj   floatr   r   r   r   r   r   r   r   r  r!  r&  rC  rZ  __all__r   r7   r6   <module>rx     s       D D D    A A !  G Q 7 7 ( 
		H	%UBII Up$ $\ %II%<<% 
% <<	%
 U\\*% % %<;ryy ;|BII $$299 $Nbii  		 'ryy 'T0
 0
f $/ $/ $/N [
! [
 [
|		  	l
 2 l
l
^ O
 2 O
O
d gr7   