UhddlZddlZddlmZddlmZmZmZmZm Z ddl Z ddl Z ddl mZddlmcmZddl mZmZmZddlmZddlmZddlmZdd lmZdd lmZmZm Z dd l!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(d dl)m*Z*m+Z+m,Z,e(jZe.Z/eGdde%Z0eGdde%Z1eGdde%Z2GddejfZ4 dIdejfde jjde jjde jjdee jjde6de6fdZ7Gdd ejfZ8Gd!d"ejfZ9Gd#d$eZ:Gd%d&ejfZ;Gd'd(ejfZ<Gd)d*ejfZ=d+Z> dJd,e jjd-e6d.e6d/e6d0e6d1e jjf d2Z?dKd3Z@d4ZAd5ZBGd6d7ejfZCe&Gd8d9e#ZDe&d:;Gd<d=eDZEGd>d?ejfZFe&d@;GdAdBeDZGe&GdCdDeDZHe&dE;GdFdGeDZIgdHZJy)LN) dataclass)AnyCallableOptionalTupleUnion)BCEWithLogitsLossCrossEntropyLossMSELoss)_calculate_fan_in_and_fan_out)ACT2FN)_prepare_4d_attention_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel) ModelOutputauto_docstringcan_return_tuplelogging) Siglip2ConfigSiglip2TextConfigSiglip2VisionConfigceZdZUdZdZeejed<dZ eejed<dZ ee ejdfed<dZ ee ejdfed<y)Siglip2VisionOutputa Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. Args: image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): The image embeddings obtained by applying the projection layer to the pooler_output. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. N image_embedslast_hidden_state. hidden_states attentions) __name__ __module__ __qualname____doc__r rtorch FloatTensor__annotations__r!r"rr#~/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/siglip2/modeling_siglip2.pyrr-sr*15L(5,,-459x 1 129=AM8E%"3"3S"89:A:>Ju00#567>r,rceZdZUdZdZeejed<dZ eejed<dZ ee ejdfed<dZ ee ejdfed<y)Siglip2TextOutputa Base class for text model's outputs that also contains a pooling of the last hidden states. Args: text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): The text embeddings obtained by applying the projection layer to the pooler_output. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. N text_embedsr!.r"r#) r$r%r&r'r0rr(r)r*r!r"rr#r+r,r-r/r/Jsr*04K%++,359x 1 129=AM8E%"3"3S"89:A:>Ju00#567>r,r/ceZdZUdZdZeejed<dZ eejed<dZ eejed<dZ eejed<dZ eejed<dZ eed<dZeed <d eefd Zy) Siglip2Outputa Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`): Contrastive loss for image-text similarity. logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`): The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text similarity scores. logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`): The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image similarity scores. text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Siglip2TextModel`]. image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Siglip2VisionModel`]. text_model_output (`BaseModelOutputWithPooling`): The output of the [`Siglip2TextModel`]. vision_model_output (`BaseModelOutputWithPooling`): The output of the [`Siglip2VisionModel`]. Nlosslogits_per_imagelogits_per_textr0r text_model_outputvision_model_outputreturncHtfdjDS)Nc3dK|]'}|dvr|nt|j)yw))r6r7N)getattrto_tuple).0kselfs r- z)Siglip2Output.to_tuple..s=  LLDGRYZ^`aRbRkRkRm m s-0)tuplekeysr?s`r-r<zSiglip2Output.to_tuples# YY[   r,)r$r%r&r'r3rr(r)r*r4r5r0r r6rr7rrr<r+r,r-r2r2gs()-D(5$$ %,48hu001837OXe//07/3K%++,304L(5,,-448186:3: %* r,r2c eZdZdeffd Zedejdejde dejfdZ dejdejdejfd Z xZ S) Siglip2VisionEmbeddingsconfigct|||_|j|_|j |_t j|j|j z|j z|j|_ |j|_ t|jdz|_ t j|j|j|_y)N) in_features out_featuresg?)super__init__rF hidden_size embed_dim patch_sizennLinear num_channelspatch_embedding num_patchesintposition_embedding_size Embeddingposition_embeddingr?rF __class__s r-rKz Siglip2VisionEmbeddings.__init__s  ++ ++!yy++doo=O  "--'*4+;+;S+@'A$"$,,t/?/?"Pr,positional_embeddingsspatial_shapes max_lengthr8cb|jd}|jd}|j}tj|||f|j|}|j dddj d}|jjdk(r|jtj}t|D]w}||\}} tj||| fddd } | j||| zjdd} | j|} | ||d || zf<| d|||| zd f<y|S) ac Resize positional embeddings to image-specific size and pad to a fixed size. Args: positional_embeddings (`torch.Tensor`): Position embeddings of shape (height, width, embed_dim) spatial_shapes (`torch.LongTensor`): Spatial shapes of shape (batch_size, 2) to resize the positional embeddings to max_length (`int`): Maximum length of the positional embeddings to pad resized positional embeddings to Returns: `torch.Tensor`: Embeddings of shape (batch_size, max_length, embed_dim) r)devicedtypercpubilinearFT)sizemode align_corners antialiasN)shaper`r(emptyr_permute unsqueezetypetofloat32rangeF interpolatereshape transpose) rZr[r\ batch_sizerM source_dtyperesulted_positional_embeddingsiheightwidthresized_embeddingss r-resize_positional_embeddingsz4Siglip2VisionEmbeddings.resize_positional_embeddingssc($))!, )//3 ,22 ). Y /(//* &!6 = =aA F P PQR S ! ' ' , , 5$9$<$.>+> ?BTUVBW *1fun.>+> ?% X(.-r, pixel_valuescJ|jjj}|j|j|}|jjj |j |j d}|j|||jd}||z}|S)aH Args: pixel_values (`torch.FloatTensor`): Pixel values of shape (batch_size, max_num_patches, num_channels * patch_size * patch_size) spatial_shapes (`List[Tuple[int, int]]`): Spatial shapes of shape (batch_size, 2) to resize the positional embeddings to )r`r^r)r\) rRweightr`rmrWrrrUr{rh)r?r|r[ target_dtype patch_embedsrZresized_positional_embeddings embeddingss r-forwardzSiglip2VisionEmbeddings.forwards++2288 ++LOO,O,OP !% 7 7 > > F F  ( ($*F*F! )-(I(I !>l>P>PQR>S)J) % "$AA r,)r$r%r&rrK staticmethodr(Tensor LongTensorrTr{r)r __classcell__rYs@r-rErEs Q2 Q8.$||8.((8.8.  8.8.tE$5$5uGWGW\a\h\hr,rEmodulequerykeyvalueattention_maskscalingdropoutc tj||jdd|z}|||z}tjj |dtj j|j}tjj|||j}tj||} | jddj} | |fS)Nr^)dimr`)ptrainingrra) r(matmulrsrO functionalsoftmaxrnrmr`rr contiguous) rrrrrrrkwargs attn_weights attn_outputs r-eager_attention_forwardrs<<s}}R'<=GL!#n4 ==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K  $$r,c eZdZdZdeeefffd Z d dejde ejde e de eje ejffdZ xZS) Siglip2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperrFct|||_|j|_|j |_|j|j z|_|j|j z|jk7r&td|jd|j d|jdz|_ |j|_ d|_ tj|j|j|_tj|j|j|_tj|j|j|_tj|j|j|_y)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).F)rJrKrFrLrMnum_attention_heads num_headshead_dim ValueErrorscaleattention_dropoutr is_causalrOrPk_projv_projq_projout_projrXs r-rKzSiglip2Attention.__init__ s  ++33$..8 ==4>> )T^^ ;MdnnM]^NN#2' ]]D( // ii? ii? ii?  $..$..A r,r"routput_attentionsr8c |j\}}}|j|}|j|}|j|} |j |||j |j jdd}|j |||j |j jdd}| j |||j |j jdd} t} |jjdk7rN|jjdk(r|rtjdnt|jj} | |||| ||j|j|j sdn |j"\} } | j%|||j'} |j)| } |sd} | | fS) z#Input shape: Batch x Time x Channelrraeagersdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rrrN)rhrrrviewrrrsrrF_attn_implementationlogger warning_oncerrrrrrrrr) r?r"rrrt seq_lengthrMqueriesrBvaluesattention_interfacerrs r-rzSiglip2Attention.forwards-:,?,?) J ++m,{{=)]+,,z:t~~t}}U__`acdeyyZOYYZ[]^_ZT^^T]]S]]^_abc(? ;; + +w 6{{//69>O##L '>dkk>^>^&_#$7     nnJJ#}}C$,, % ! \"))*j)LWWY mmK0  LL((r,NF)r$r%r&r'rrrrKr(rrboolrrrrs@r-rrs{GBu%8:K%KLB.26,1 -)||-)!.-)$D> -) u||Xell33 4 -)r,rcVeZdZfdZdej dej fdZxZS) Siglip2MLPct|||_t|j|_t j|j|j|_ t j|j|j|_ yN) rJrKrFr hidden_act activation_fnrOrPrLintermediate_sizefc1fc2rXs r-rKzSiglip2MLP.__init__Psd  #F$5$5699V//1I1IJ99V55v7I7IJr,r"r8cl|j|}|j|}|j|}|Sr)rrr)r?r"s r-rzSiglip2MLP.forwardWs4/ **=9 / r,)r$r%r&rKr(rrrrs@r-rrOs$KU\\ellr,rc eZdZdeeefffd Z ddejdejde e de ejfdZ xZS) Siglip2EncoderLayerrFcDt||j|_t j |j|j |_t||_ t j |j|j |_ t||_ y)Neps) rJrKrLrMrO LayerNormlayer_norm_eps layer_norm1r self_attn layer_norm2rmlprXs r-rKzSiglip2EncoderLayer.__init___sm ++<<F $ u  ! $r,rc xeZdZdZdeffd Ze d deejdee dee de fdZ xZ S) Siglip2Encoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`Siglip2EncoderLayer`]. Args: config: Siglip2Config rFct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}wr) rJrKrFrO ModuleListronum_hidden_layersrlayersgradient_checkpointing)r?rF_rYs r-rKzSiglip2Encoder.__init__sP  mm%PVPhPhJi$jQ%8%@$jk &+#%ksA#rroutput_hidden_statesr8c||n|jj}||n|jj}|rdnd}|rdnd}|}|jD]&}|r||fz}||||} | d}|s|| dfz}(|r||fz}t |||S)ad Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr+)rrr)r!r"r#)rFrrrr) r? inputs_embedsrrrencoder_statesall_attentionsr" encoder_layer layer_outputss r-rzSiglip2Encoder.forwards<2C1N-TXT_T_TqTq$8$D $++JjJj  40d% ![[ FM#!/=2B!B)"3M *!,M !/=3C2E!E F +}.>>N+(%  r,NNN)r$r%r&r'rrKrrr(rrrrrrs@r-rrsl,},26,0/3 < !.< $D> < 'tn <  < < r,rceZdZdeffd Zee d dejdejdejde e de e de f d ZxZS) Siglip2VisionTransformerrFcrt|||_|j}t ||_t ||_tj||j|_ t|dsdn |j|_|jrt||_|j"dk(|_y)Nrvision_use_headTflash_attention_2)rJrKrFrLrErrencoderrOrrpost_layernormhasattrruse_head$Siglip2MultiheadAttentionPoolingHeadheadr_use_flash_attention_2r?rFrMrYs r-rKz!Siglip2VisionTransformer.__init__s  && 1&9%f-  ll9&:O:OP$+F4E$FFLbLb == * 'tn *  $* * r,rc eZdZdeffd Z ddeejdeejdeejdejfdZ xZ S) Siglip2TextEmbeddingsrFcNt||j}tj|j ||_tj|j||_|jdtj|jjddy)N position_ids)rr^F) persistent) rJrKrLrOrV vocab_sizetoken_embeddingmax_position_embeddingsrWregister_bufferr(arangeexpandrs r-rKzSiglip2TextEmbeddings.__init__s && !||F,=,=yI"$,,v/M/My"Y  ELL)G)GHOOPWXej  r, input_idsrrr8c8||jdn|jd}|jjjd}||kDrtd|d|||jddd|f}||j |}|j|}||z}|S)Nr^rrzRSequence length must be less than max_position_embeddings (got `sequence length`: z and max_position_embeddings: )rhrWr~rrr)r?rrrrmax_position_embeddingposition_embeddingsrs r-rzSiglip2TextEmbeddings.forward(s -6,AY__R(}GZGZ[]G^ !%!8!8!?!?!E!Ea!H . .d,<=S.norm_cdfFs(dhhq499S>122c99r,razjmean is more than 2 std from [a, b] in nn.init.trunc_normal_. The distribution of values may be incorrect.) stacklevelrr )minmax) warningswarnuniform_erfinv_mul_r r add_clamp_)tensormeanstdabrlus r-_trunc_normal_r Cs: q1s7{q1s7{ 2  ;  !d(c!"A!d(c!"A OOAEAIq1uqy) NN KKdiin$% KK MMaQMr,rrrrrr8ctj5t|dd|||j|j |dddy#1swYyxYw)anFills the input Tensor with values drawn from a truncated normal distribution. The values are effectively drawn from the normal distribution :math:`\mathcal{N}( ext{mean}, ext{std}^2)` with values outside :math:`[a, b]` redrawn until they are within the bounds. The method used for generating the random values works best when :math:`a \leq ext{mean} \leq b`. NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0 and the result is subsequently scaled and shifted by the mean and std args. Args: tensor: an n-dimensional `torch.Tensor` mean: the mean of the normal distribution std: the standard deviation of the normal distribution a: the minimum cutoff value b: the maximum cutoff value rrN)r(no_gradr rr)rrrrrs r-trunc_normal_tf_r#gsI* $vq#q!, Cd#$$$s 0AAc,t|\}}|dk(r|}n|dk(r|}n |dk(r||zdz }|z }|dk(r$t|tj|dz y|dk(rCt j 5|j tj|dddy|d k(rItjd |z}t j 5|j| |dddytd |#1swYyxYw#1swYyxYw) Nfan_infan_outfan_avgratruncated_normalg۶%?rnormaluniformr zinvalid distribution ) r r#r r r(r"normal_rr) rrre distributionr%r&denomvariancebounds r-variance_scaling_r1s3F;OFG x    '!Q&u}H))TYYx%8;N%NO  ! ]]_ 4 NNtyy2N 3 4 4  " !h,' ]]_ + OOUFE * + +0?@@ 4 4 + +s3&C>D >D Dc t|ddy)Nr%r(rer-r1rs r- lecun_normal_r6sf8:LMr,c t|ddy)Nr%r*r3r4r5s r-default_flax_embed_initr8sf8(Cr,ceZdZdeffd Zee d deejdeejdeejdee dee de f d Z xZ S) Siglip2TextTransformerrFcFt|||_|j}t ||_t ||_tj||j|_ tj||j|_|jdk(|_y)Nrr)rJrKrFrLrrrrrOrrfinal_layer_normrPprojection_sizerrrrs r-rKzSiglip2TextTransformer.__init__s}  && /7%f- " YF . 'tn .  $. . r,r:c.eZdZeZdZdZgdZdZdZ dZ y)Siglip2PreTrainedModelsiglip2T)rrrErrc t|trt|jtr |jjj n|jj }t jj|jjdtj|z yt|t jrt|jyt|trt jj!|j"jt jj!|j$jt jj!|j&jt jj!|j(jt jj+|j"j,t jj+|j$j,t jj+|j&j,t jj+|j(j,yt|t.rt jj!|j0jt jj!|j2jt jj|j0j,dt jj|j2j,dyt|t4rt jj!|j6j8t jj!|j:j<j8t jj+|j:j>j8yt|t@rrtCjDtCjFd}|jHj8jK||jLj8jOyt|tPrnt jj|jRj|jjj dz|jjTzyt|t jVt jXfrLt[|j|j,*t jj+|j,yyt|t j\rJ|j,j8jO|jj8jKdyy)zInitialize the weightsrr)gư>rrN)/ isinstancerErFr vision_configrLrOinitr,rWr~npr rVr8rxavier_uniform_rrrrzeros_biasrrrrprobedata attentionin_proj_weight in_proj_bias Siglip2Modelr(logr logit_scalefill_ logit_biaszero_Siglip2ForImageClassification classifierinitializer_factorrPConv2dr6r)r?rrylogit_scale_inits r- _init_weightsz$Siglip2PreTrainedModel._init_weightss f5 6dkk=9 ))55[[,,  GGOOF55<  -$yyc):;     # # ) )*: ;    " " ( ( *  = > GGOO!!((KK--994?$++B`B``  BII 6 7 &-- ({{&v{{+'  - KK   " " $ MM   $ $S ).r,N) r$r%r&r config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_flash_attn_2_supports_sdpar]r+r,r-rCrCs- L!&*#"N,*r,rCzL The text model from Siglip2 without any head or projection on top. ) custom_introceZdZeZdeffd ZdejfdZdZ e e d de e jde e jde e jd e ed e edef d ZxZS) Siglip2TextModelrFcdt||t||_|j yr)rJrKr: text_model post_initrXs r-rKzSiglip2TextModel.__init__(s&  08 r,r8cB|jjjSrrhrrrCs r-get_input_embeddingsz%Siglip2TextModel.get_input_embeddings.s))999r,c:||jj_yrrk)r?rs r-set_input_embeddingsz%Siglip2TextModel.set_input_embeddings1s5:""2r,rrrrrc.|j|||||S)a Examples: ```python >>> from transformers import AutoTokenizer, Siglip2TextModel >>> model = Siglip2TextModel.from_pretrained("google/siglip2-base-patch16-224") >>> tokenizer = AutoTokenizer.from_pretrained("google/siglip2-base-patch16-224") >>> # important: make sure to set padding="max_length" as that's how the model was trained >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled (EOS token) states ```rrrrr)rh)r?rrrrrs r-rzSiglip2TextModel.forward4s)6)%/!5   r,rA)r$r%r&rr^rKrOModulerlrnrrrr(rrrrrrs@r-rfrf s %L0 :bii:;-115/3,0/3  ELL) !. u||,  $D>  'tn   $  r,rfceZdZdZdeffd ZddejdeejdejfdZ xZ S) rzMultihead Attention Pooling.rFct|tjt j dd|j |_tjj|j |jd|_ tj|j |j|_ t||_|j|_y)NrT) batch_firstr)rJrKrO Parameterr(randnrLrMMultiheadAttentionrrOrr layernormrrrrXs r-rKz-Siglip2MultiheadAttentionPoolingHead.__init__[s \\%++aF4F4F"GH 44V5G5GIcIcqu4vf&8&8f>S>STf%33r, hidden_staterr8c|jd}|jj|dd}|f|jd|jd}}t||j|}|jd|j |d}|j d||}|j||||d}|}|j|}||j|z}|dddfS)Nrrr^) attn_mask) rhrMrepeatrr`rrrrOrxr)r?ryrrtrM target_len source_lenrs r-rz,Siglip2MultiheadAttentionPoolingHead.forwardds!''*  !!*a3  %%*[[^\5G5G5J J7 HZHZ\fgN+221dnnjRSTN+33B JON~~e\>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, Siglip2VisionModel >>> model = Siglip2VisionModel.from_pretrained("google/siglip2-base-patch16-224") >>> processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled features ```r|rr[rr)r)r?r|rr[rrs r-rzSiglip2VisionModel.forwards,F  %/)/!5 !  r,r)r$r%r&rr^main_input_namerKrOrqrlrrr(r)rrrrrrrrs@r-rrws 'L$O2 ' 'tn '  $' ' r,rcveZdZeZdeffd Ze ddeejdeejdeejdee dee dejf d Z e dd eejd eejd eejdee dee dejf d Zee ddeejd eejd eejd eejdeejdeejdee dee dee defdZxZS)rRrFct||t|jts"t dt |jdt|jts"t dt |jd|j}|j}tj|}tj|}|j|_ |j|_ tjt!j"d|_tjt!j"d|_|j)y)NzNconfig.text_config is expected to be of type Siglip2TextConfig but is of type .zRconfig.vision_config is expected to be of type Siglip2VisionConfig but is of type r)rJrKrF text_configr TypeErrorrlrGrrf _from_configrrhrrOrur(rvrTrVri)r?rFrrGrhrrYs r-rKzSiglip2Model.__init__s  &,,.?@++,-Q0  &..0CD--./q2  (( ,, &22;? )66}E %//(55<< A7,,u{{1~6 r,rrrrrr8c||n|jj}||n|jj}|j|||||}|j}|S)aM Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`Siglip2TextModel`]. Examples: ```python >>> from transformers import AutoTokenizer, AutoModel >>> import torch >>> model = AutoModel.from_pretrained("google/siglip2-base-patch16-224") >>> tokenizer = AutoTokenizer.from_pretrained("google/siglip2-base-patch16-224") >>> # important: make sure to set padding="max_length" as that's how the model was trained >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding="max_length", return_tensors="pt") >>> with torch.no_grad(): ... text_features = model.get_text_features(**inputs) ```rp)rFrrrhr)r?rrrrr text_outputsr@s r-get_text_featureszSiglip2Model.get_text_featuressr:2C1N-TXT_T_TqTq$8$D $++JjJj 48??)%/!5 4C4 %22 r,r|rr[c||n|jj}||n|jj}|j|||||}|j}|S)a pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`Siglip2VisionModel`]. Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, AutoModel >>> import torch >>> model = AutoModel.from_pretrained("google/siglip2-base-patch16-224") >>> processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... image_features = model.get_image_features(**inputs) ``` r)rFrrrr)r?r|rr[rrvision_outputsr@s r-get_image_featureszSiglip2Model.get_image_features suP2C1N-TXT_T_TqTq$8$D $++JjJj 6:5F5F%/)/!5 6G6 '44 r, return_lossc ||n|jj}| | n|jj} |j||||| } |j ||||| } | j } | j } | | j dddz } | | j dddz } tj| | jj| j}|jj| j|jj| j}}||jz|z}|j}d}|rtj|j!d|j }tj"| d|zz}tj$j&j)||z}tj*|d  }|j-}t/|||| | | | S) a pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, AutoModel >>> import torch >>> model = AutoModel.from_pretrained("google/siglip2-base-patch16-224") >>> processor = AutoProcessor.from_pretrained("google/siglip2-base-patch16-224") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> texts = ["a photo of 2 cats", "a photo of 2 dogs"] >>> # important: we pass `padding=max_length` since the model was trained with this >>> inputs = processor(text=texts, images=image, padding="max_length", return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> logits_per_image = outputs.logits_per_image >>> probs = torch.sigmoid(logits_per_image) # these are the probabilities >>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'") 31.9% that image 0 is 'a photo of 2 cats' ``` Nrrprar^T)rrkeepdimr)r_r)r3r4r5r0r r6r7)rFrrrrhrnormr(rtrmr_rTrVexpeyerd ones_likerOr logsigmoidsumrr2)r?rr|rr[rrrrrrrr r0r5rTrVr4r3rm1_diag1logliknlls r-rzSiglip2Model.forwardBsd2C1N-TXT_T_TqTq$8$D $++JjJj 6:5F5F%/)/!5 6G6 48??)%/!5 4C4 &33 "00 $l&7&7!T&7&RR !K$4$4qb$$4$OO  ,,{LNN4D4G4G HZHZ4[\"&"2"2"5"5k6H6H"I4??K]K]^i^p^pKqZ )KOO,== J*,,. ))O003O + 'tn +   ++Z597;59,0/3 6u0016'u||46!!1!12 6 $D> 6 'tn 6   66p15487;591537&*,0/3e E,,-e u001e 'u||4 e !!1!12 e !. e u//0e d^e $D>e 'tne  e e r,rRz Siglip2 vision encoder with an image classification head on top (a linear layer on top of the pooled final hidden states of the patch tokens) e.g. for ImageNet. ceZdZdZdeddffd Zee d dee jdee jdee jdee jd ee d ee de fd ZxZS) rXr|rFr8Nc~t|||j|_tj |j }|j |_|jdkDr4tj|j j|jntj|_ |jy)Nr) rJrK num_labelsrrrGrrOrPrLIdentityrYri)r?rFrrYs r-rKz&Siglip2ForImageClassification.__init__s   ++*66v7K7KL (55OUN_N_bcNcBIIf**668I8I Jikititiv  r,rr[labelsrrc8||n|jj}||n|jj}|j|||||}|j}|Q|dj |j } tj|| zdtj| dz }ntj|d}|j|} d} ||j | j }|jj|jdk(rd|j_ nl|jdkDrL|jtjk(s|jtjk(rd|j_ nd|j_ |jjdk(rIt!} |jdk(r&| | j#|j#} n| | |} n|jjdk(r=t%} | | j'd |j|j'd } n,|jjdk(rt)} | | |} t+| | |j,|j. S) a pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*): Mask to avoid performing attention on padding pixel indices. spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`): Tensor containing the spatial dimensions (height, width) of the input images. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Examples: ```python >>> from transformers import AutoImageProcessor, Siglip2ForImageClassification >>> import torch >>> from PIL import Image >>> import requests >>> torch.manual_seed(3) # doctest: +IGNORE_RESULT >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> # note: we are loading a `Siglip2Model` from the hub here, >>> # so the head will be randomly initialized, hence the predictions will be random if seed is not set above. >>> image_processor = AutoImageProcessor.from_pretrained("google/siglip2-base-patch16-224") >>> model = Siglip2ForImageClassification.from_pretrained("google/siglip2-base-patch16-224") >>> inputs = image_processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> logits = outputs.logits >>> # model predicts one of the two classes >>> predicted_class_idx = logits.argmax(-1).item() >>> print("Predicted class:", model.config.id2label[predicted_class_idx]) Predicted class: LABEL_1 ``` N)rr[rr).Nrr regressionsingle_label_classificationmulti_label_classificationr^)r3logitsr"r#)rFrrrr!rmr_r(rrrY problem_typerr`longrTr squeezer rr rr"r#) r?r|rr[rrrrsequence_output pool_maskrr3loss_fcts r-rz%Siglip2ForImageClassification.forwards8^2C1N-TXT_T_TqTq$8$D $++JjJj /3.?.? /)/!5 /@/ "33 +,Y7::?;Q;QRI#ii)(CKeiiXaghNiiO#jja@O1  YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+- B @&++b/R))-II,./$!//))   r,)NNNNNN)r$r%r&rrrKrrrr(rrrrrrrs@r-rXrXs%O}$047;59)-,0/3d u||,d 'u||4d !!1!12 d & d $D> d 'tnd  d d r,rX)rRrCrfrrX)r)rrgr )rr%r*)Kr r dataclassesrtypingrrrrrnumpyrIr(torch.nnrOtorch.nn.functionalrrpr r r torch.nn.initr activationsrmodeling_attn_mask_utilsrmodeling_layersrmodeling_outputsrrrmodeling_utilsrrutilsrrrrconfiguration_siglip2rrr get_loggerr$rrr/r2rqrErfloatrrrrrrrr r#r1r6r8r:rCrfrrrRrX__all__r+r,r-rs* !88 AA7!B9bbFKKXX   H % ?+? ?8 ? ? ?8 ! K!  ! HbbiibX% II% <<% % << % U\\* %  %%.D)ryyD)N  -4-`M RYYM `: ryy: z%BII%P! J\_$ LL$ %$27$BG$SX$ \\$4A2ND< RYY< ~;*_;*;*| 0 -0  0 f"299"> 8 /8  8 vq )q q h { $:{  { | r,