
    Uh                       d dl Z d dlmZ d dlmZmZmZmZmZm	Z	m
Z
 d dlZd dlZd dlmZ d dlmZ ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z> ddl?m@Z@mAZAmBZBmCZCmDZDmEZE ddlFmGZGmHZH ddlImJZJ ddlKmLZL ddlMmNZNmOZOmPZP  e6       rd dlZd dlQmZ d dlRmc mSZT d dlUZ e7       rd dlVZVddlWmXZX ddl9mYZYmZZZ  e8j                  e\      Z] G d deL      Z^ G d  d!e>      Z_ G d" d#eX      Z`e4 G d$ d%e0             Zae G d& d'e-             Zbe G d( d)eG             Zce G d* d+eH             Zd G d, d-eP      Ze G d. d/ej                        Zg G d0 d1ej                        Zh G d2 d3eO      Zi G d4 d5eN      Zj G d6 d7e<      Zk G d8 d9ej                        Zl G d: d;eE      Zm G d< d=eD      Zn G d> d?eB      Zo G d@ dAeC      Zp G dB dCej                        Zq G dD dEej                        Zr G dF dGeAej                        Zs G dH dIej                        Zt G dJ dKe@      Zu G dL dMej                        Zv G dN dOej                        Zw e4dPQ       G dR dSea             Zx G dT dUeae      Zy G dV dWe      Zzg dXZ{y)Y    N)	dataclass)CallableDictIterableListOptionalTupleUnion)nn)BlipImageProcessor   )ACT2FN)Cache)%ClassifierFreeGuidanceLogitsProcessorGenerationMixinGenerationModeLogitsProcessorList)GenerateDecoderOnlyOutput)BatchFeatureget_size_dict)resizeto_channel_dimension_format)ChannelDimension
ImageInputPILImageResamplingget_image_sizeinfer_channel_dimension_formatmake_list_of_imagesto_numpy_array)FlashAttentionKwargs)ModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringcan_return_tupleis_torch_availableis_vision_availablelogging   )	AutoModel)Blip2VisionModel)ChameleonVQVAEConfig)ChameleonVQVAEChameleonVQVAEEncoderChameleonVQVAEEncoderAttnBlock#ChameleonVQVAEEncoderConvDownsample ChameleonVQVAEEncoderResnetBlockChameleonVQVAEVectorQuantizer)IdeficsBaseModelOutputWithPastIdeficsCausalLMOutputWithPast)eager_attention_forward)SiglipVisionConfig)SiglipEncoderSiglipEncoderLayerSiglipVisionEmbeddings)PretrainedConfig)CONFIG_MAPPING
AutoConfigc                   P     e Zd ZdZdZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z xZS )JanusVisionConfiga
  
    This is the configuration class to store the configuration of a [`JanusVisionModel`]. It is used to instantiate a
    `JanusVisionModel` according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
    Args:
        hidden_size (`int`, *optional*, defaults to 1024):
            Dimensionality of the encoder layers and the pooler layer.
        num_hidden_layers (`int`, *optional*, defaults to 24):
            Number of hidden layers in the Transformer encoder.
        num_attention_heads (`int`, *optional*, defaults to 16):
            Number of attention heads for each attention layer in the Transformer encoder.
        num_channels (`int`, *optional*, defaults to 3):
            The number of input channels.
        patch_size (`int`, *optional*, defaults to 16):
            The size (resolution) of each patch.
        image_size (`int`, *optional*, defaults to 384):
            The size (resolution) of each image.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for attention weights.
        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the layer normalization layers.
        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"selu"`, and `"gelu_new"` are supported.
        mlp_ratio (`float`, *optional*, defaults to 4.0):
            Ratio of MLP hidden dimensionality to embedding dimensionality.
        attention_bias (`bool`, *optional*, defaults to `True`):
            Whether to add a bias to the queries, keys, and values in the attention layers.
        hidden_dropout_rate (`float`, *optional*, defaults to 0.0):
            The dropout probability for fully connected layers in the encoder.
        projection_dim (`int`, *optional*, defaults to 2048):
            Dimensionality of the MLP projection head.
        projection_dropout (`float`, *optional*, defaults to 0.0):
            Dropout probability for the projection layer.
        use_qk_norm (`bool`, *optional*, defaults to `False`):
            Whether to normalize the query and key matrices.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated normal initializer for initializing all weight matrices.
        depth (`int`, *optional*, defaults to 2):
            Number of hidden layers in the aligner module.
        num_image_tokens (`int`, *optional*, defaults to 576):
            Number of image tokens.
    janus_vision_modelvision_configc                     t        |   d|||||||||	d	| | `|
| _        || _        || _        || _        || _        || _        || _	        || _
        || _        y )N)	hidden_sizenum_hidden_layersnum_attention_headsnum_channels
patch_size
image_sizeattention_dropoutlayer_norm_eps
hidden_act )super__init__intermediate_size	mlp_ratioattention_biashidden_dropout_rateprojection_dimprojection_dropoutuse_qk_norminitializer_rangedepthnum_image_tokens)selfrC   rD   rE   rF   rG   rH   rI   rJ   rK   rP   rQ   rR   rS   rT   rU   rV   rW   rX   kwargs	__class__s                       y/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/janus/modular_janus.pyrN   zJanusVisionConfig.__init__   s    , 	 	
#/ 3%!!/)!	
 	
 "",#6 ,"4&!2
 0    )i         r   r_   i          ư>gelug      @Tr`      r`   F{Gz?r*   i@  )__name__
__module____qualname____doc__
model_typebase_config_keyrN   __classcell__r[   s   @r\   r?   r?   Q   sW    ,\ &J%O ',1 ,1r]   r?   c                   |     e Zd ZdZddddddddg d	d
dddd
ddfdededededededededee   dedef fdZ xZ	S )JanusVQVAEConfiga:
  
    This is the configuration class to store the configuration of a [`JanusVQVAEModel`]. It is used to instantiate a
    `JanusVQVAEModel` according to the specified arguments, defining the model architecture.
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information. Instantiating a
    configuration with the defaults will yield a similar configuration to the VQModel of the
    [deepseek-community/Janus-Pro-1B](https://huggingface.co/deepseek-community/Janus-Pro-1B).

    Args:
        embed_dim (`int`, *optional*, defaults to 8):
            Dimensionality of each embedding vector.
        num_embeddings (`int`, *optional*, defaults to 16384):
            Number of codebook embeddings.
        double_latent (`bool`, *optional*, defaults to `False`):
            Whether to use double z channels.
        latent_channels (`int`, *optional*, defaults to 256):
            Number of channels for the latent space.
        num_patches (`int`, *optional*, defaults to 32):
            Num of patches the input images can be divided into.
        in_channels (`int`, *optional*, defaults to 3):
            Number of input channels.
        out_channels (`int`, *optional*, defaults to 3):
            Number of out channels.
        base_channels (`int`, *optional*, defaults to 128):
            Base channel count.
        channel_multiplier (`List[int]`, *optional*, defaults to `[1, 1, 2, 2, 4]`):
            Channel multipliers for each resolution.
        num_res_blocks (`int`, *optional*, defaults to 2):
            Number of residual blocks.
        dropout (`float`, *optional*, defaults to 0.0):
            Dropout rate.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        projection_dim (`int`, *optional*, defaults to 2048):
            Dimensionality of the MLP projection head.
        num_hidden_layers (`int`, *optional*, defaults to 2):
            Number of hidden layers in VAVAE MLP Connecter module.
        hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"silu"` and `"gelu_new"` are supported.
        image_token_embed_dim (`int`, *optional*, defaults to 2048):
            Dimension of image embeddings. It should be same as the dimensionality of text embeddings.
       i @  F       r      )   rs   r*   r*      r*   r`   rd   rc   rb   	embed_dimnum_embeddingsdouble_latentlatent_channelsnum_patchesin_channelsout_channelsbase_channelschannel_multipliernum_res_blocksdropoutc                     t        |   d|||||||	|
||d
| || _        || _        || _        || _        || _        || _        | `| `	| `
y )N)
ru   rv   rw   rx   rz   r|   r}   r~   r   rV   rL   )rM   rN   ry   r{   rS   rD   rK   image_token_embed_dim
resolutionattn_resolutions	attn_type)rY   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   rV   rS   rD   rK   r   rZ   r[   s                     r\   rN   zJanusVQVAEConfig.__init__   s    ( 	 	
)'+#'1)/	
 	
 '(,!2$%:"O!Nr]   )
re   rf   rg   rh   intboolr   floatrN   rk   rl   s   @r\   rn   rn      s    *\ ##" (7"#** * 	*
 * * * * * !I* * * *r]   rn   c                   <     e Zd ZdZdZeeedZ	 	 	 	 d fd	Z	 xZ
S )JanusConfiga;  
    This is the configuration class to store the configuration of a [`JanusModel`]. It is used to instantiate an
    Janus model according to the specified arguments, defining the model architecture. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the Janus-1B or Janus-7B models.

    e.g. [deepseek-community/Janus-Pro-1B](https://huggingface.co/deepseek-community/Janus-Pro-1B) or
    [deepseek-community/Janus-Pro-7B](https://huggingface.co/deepseek-community/Janus-Pro-7B)

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.

    Args:
        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `LlamaConfig`):
            The config object or dictionary of the text backbone.
        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `JanusVisionConfig`):
            The config object or dictionary of the vision backbone.
        vq_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `JanusVQVAEConfig`):
            The config object or dictionary of the VQVAE backbone.
        image_token_id (`int`, *optional*, defaults to 100581):
            Token index of a placeholder image token.

    Example:

    ```python
    >>> from transformers import JanusForConditionalGeneration, JanusConfig, JanusVisionConfig, JanusVQVAEConfig, LlamaConfig

    >>> # Initializing a Janus vision config
    >>> vision_config = JanusVisionConfig()

    >>> # Initializing a Llama config
    >>> text_config = LlamaConfig()

    >>> # Initializing a VQ config
    >>> vq_config = JanusVQVAEConfig()

    >>> # Initializing a Janus Pro 1B style configuration
    >>> configuration = JanusConfig(vision_config=vision_config, text_config=text_config, vq_config=vq_config)

    >>> # Initializing a model from the Janus Pro 1B style configuration
    >>> model = JanusForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```janus)text_configrA   	vq_configc                    t        |t              r,|j                  dd      |d<   t        |d      d	i || _        nY|(t
        j                  d       t        d          | _        n/t        |t              r|| _        nt        dt        |             |%t
        j                  d       t               | _        nPt        |t              rt        d	i || _        n/t        |t              r|| _        nt        dt        |             |%t
        j                  d       t               | _        nPt        |t              rt        d	i || _        n/t        |t              r|| _        nt        dt        |             | j                  j                  | j                  j                  z  | j                  _        || _        t%        | L  d	i | y )
Nri   llamaz7`text_config` is None. Initializing with default valueszTInvalid type for `text_config`. Must be either `dict` or `LlamaConfig`. Type found: zK`vision_config` is None. Initializing with default JanusVisionConfig valuesz\Invalid type for `vision_config`. Must be either `dict` or `JanusVisionConfig`. Type found: zF`vq_config` is None. Initializing with default JanusVQVAEConfig valueszWInvalid type for `vq_config`. Must be either `dict` or `JanusVQVAEConfig`. Type found: rL   )
isinstancedictgetr<   r   loggerinfor;   
ValueErrortyper?   rA   rn   r   rH   rG   ry   image_token_idrM   rN   )rY   r   rA   r   r   rZ   r[   s         r\   rN   zJanusConfig.__init__A  s    k4((3g(NK%-k,.GHW;WD KKQR-g68D%56*D  $[ 124 
  KKef!2!4Dt,!2!C]!CD'89!.D  $] 346 
 KK`a-/DN	4(-:	:DN	#34&DN  $Y02  &*%7%7%B%BdFXFXFcFc%c","6"r]   )NNNi )re   rf   rg   rh   ri   r=   r?   rn   sub_configsrN   rk   rl   s   @r\   r   r     s8    +Z J!*%K 5# 5#r]   r   c                   D    e Zd ZeZdZdZdgZddgZdZ	dZ
dZdZdZdZd Zy)	JanusPreTrainedModelmodelTLlamaDecoderLayerpast_key_valuescausal_maskFc                    t        | j                  d      r | j                  j                  j                  n| j                  j                  }t	        |t
        j                  t
        j                  f      rY|j                  j                  j                  d|       |j                  %|j                  j                  j                          y y t	        |t
        j                  t
        j                  f      rJ|j                  j                  j                          |j                  j                  j                  d       y t	        |t
        j                         rf|j                  j                  j                  d|       |j"                  2|j                  j                  |j"                     j                          y y y )NrA   r`   )meanstd      ?)hasattrconfigrA   rV   r   r   LinearConv2dweightdatanormal_biaszero_	GroupNorm	LayerNormfill_	Embeddingpadding_idx)rY   moduler   s      r\   _init_weightsz"JanusPreTrainedModel._init_weights  s;    t{{O4 KK%%77.. 	
 fryy"))45MM&&CS&9{{&  &&( 'r|| <=KK""$MM$$S)-MM&&CS&9!!-""6#5#56<<> . .r]   N)re   rf   rg   r   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_quantized_cache_supports_cache_class_supports_static_cache!_supports_param_buffer_assignmentr   rL   r]   r\   r   r   y  sO    L&*#,-#4m"D!N $ !(-%?r]   r   c                   \    e Zd ZU dZdZeej                     ed<   dZ	ej                  ed<   y)JanusVQVAEOutputaM  
    Base class for Janus VQ-VAE mode model outputs.
    Args:
        decoded_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            Reconstructed pixel values after encoding and decoding the input.
        embedding_loss (`torch.FloatTensor`):
            Embedding loss.
    Ndecoded_pixel_valuesembedding_loss)
re   rf   rg   rh   r   r   torchFloatTensor__annotations__r   rL   r]   r\   r   r     s/     9=(5#4#45<(,NE%%,r]   r   c                       e Zd Zy)JanusBaseModelOutputWithPastNre   rf   rg   rL   r]   r\   r   r         r]   r   c                       e Zd Zy)JanusCausalLMOutputWithPastNr   rL   r]   r\   r   r     r   r]   r   c                   J    e Zd Zddej                  dedej                  fdZy)JanusVisionEmbeddingspixel_valuesinterpolate_pos_encodingreturnc                 X   |j                   \  }}}}| j                  j                  j                  }| j                  |j	                  |            }|j                  d      j                  dd      }|r| j                  |||      }	n| j                  | j                        }	||	z   }|S )Ndtyper*   rs   )
shapepatch_embeddingr   r   toflatten	transposer   position_embeddingposition_ids)
rY   r   r   _heightwidthtarget_dtypepatch_embeds
embeddings
pos_embedss
             r\   forwardzJanusVisionEmbeddings.forward  s    *001fe++2288++LOO,O,OP!))!,66q!<
#66z65QJ001B1BCJ*,
r]   N)F)re   rf   rg   r   Tensorr   r   rL   r]   r\   r   r     s'    ELL D ]b]i]i r]   r   c            
            e Zd ZdZdef fdZ	 	 d	dej                  deej                     deej                     de	e
   fdZ xZS )
JanusVisionAttentionz(Attention Class for Janus Vision Encoderr   c                 F   t         |           || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                   d| j                   d      | j                  dz  | _	        |j                  | _
        |j                  }|j                  }d| _        d| _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                  | j                  z  |j"                        | _        t        j                   | j                  | j                        | _        |dkD  rt        j,                  |      nt        j.                         | _        |rt        j0                  | j                        nt        j.                         | _        |r%t        j0                  | j                        | _        y t        j.                         | _        y )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      Frs   r   r   )rM   rN   r   rC   ru   rE   	num_headshead_dimr   scalerI   rT   rU   	is_causalnum_key_value_groupsr   r   rQ   q_projk_projv_projprojection_layerDropoutIdentityr   q_normk_norm)rY   r   proj_dropoutqk_normr[   s       r\   rN   zJanusVisionAttention.__init__  s   ++33$..8==4>>)T^^;MdnnM] ^NN#2'  ]]D(
!'!9!900$$ %&!ii0NU[UjUjkii0NU[UjUjkii0NU[UjUjk "		$..$.. I>JQ>N"**\":TVT_T_Ta6=bll4>>22;;=6=bll4>>22;;=r]   hidden_statesattention_maskoutput_attentionsrZ   c                    |j                         \  }}}| j                  |      }| j                  |      }	| j                  |      }
|j	                  d| j
                  | j                        }| j                  |      }|	j	                  d| j
                  | j                        }	| j                  |	      }	|j	                  ||| j
                  | j                        j                  dd      }|	j	                  ||| j
                  | j                        j                  dd      }	|
j                  ||| j
                  | j                        j                  dd      }
t        }| j                  j                  dk7  r^| j                  j                  dk(  r(|j                  dd      rt        j!                  d       nt"        | j                  j                     } || ||	|
|f| j$                  sd	n| j&                  | j(                  | j*                  d
|\  }}|j	                  ||| j,                        }| j/                  |      }| j1                  |      }|r||f}|S |d f}|S )Nrs   r*   eagersdpar   Fz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r`   )r   scalingr   )sizer   r   r   reshaper   r   r   r   r   viewr6   r   _attn_implementationr   r   warning_oncer"   trainingrI   r   r   ru   r   rT   )rY   r   r   r   rZ   
batch_sizeseq_lenr   query_states
key_statesvalue_statesattention_interfaceattn_outputattn_weightsoutputoutputss                   r\   r   zJanusVisionAttention.forward  s0    "/!3!3!5
GQ{{=1[[/
{{=1#++BN{{<0''DNNDMMJ
[[,
#++JQUQ^Q^_iijkmno''
GT^^T]][eefgijk
#((Wdnndmm\ffghjkl(?;;++w6{{//69fjjI\^c>d##L
 '>dkk>^>^&_#$7
%
  $}}C$2H2HJJnn
%
 
%
!\ "))*gt~~N&&{3((0,=6<( EKD>r]   )NN)re   rf   rg   rh   r?   rN   r   r   r   r$   r    r   rk   rl   s   @r\   r   r     se    2Q0 Q@ 2648	2||2 !.2 $ELL1	2
 -.2r]   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )JanusVisionMLPr   c                    t         |           || _        t        |j                  |j
                  z        | _        t        |j                     | _	        t        j                  |j                  | j                        | _        t        j                  | j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                        | _        y N)rM   rN   r   r   rC   rP   rO   r   rK   activation_fnr   r   fc1fc2r   rR   dropout1dropout2rY   r   r[   s     r\   rN   zJanusVisionMLP.__init__  s    !$V%7%7&:J:J%J!K#F$5$5699V//1G1GH99T33V5G5GH

6#=#=>

6#=#=>r]   r   r   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r  )r  r  r  r  r  rY   r   s     r\   r   zJanusVisionMLP.forward$  sP    /**=9m4/m4r]   )	re   rf   rg   r?   rN   r   r   r   rk   rl   s   @r\   r  r    s+    ?0 ?U\\ ell r]   r  c                   $     e Zd Zdef fdZ xZS )JanusVisionEncoderLayerr   c                 R   t         |           || _        |j                  | _        t        |      | _        t        j                  | j                  |j                        | _
        t        j                  | j                  |j                        | _        t        |      | _        y )N)eps)rM   rN   r   rC   ru   r   	self_attnr   r   rJ   layer_norm1layer_norm2r  mlpr  s     r\   rN   z JanusVisionEncoderLayer.__init__.  st    ++-f5<<F<Q<QR<<F<Q<QR!&)r]   re   rf   rg   r?   rN   rk   rl   s   @r\   r  r  -  s    *0 * *r]   r  c                   $     e Zd Zdef fdZ xZS )JanusVisionEncoderr   c                     t         |   |       t        j                  t	        |j
                        D cg c]  }t        |       c}      | _        y c c}w r  )rM   rN   r   
ModuleListrangerD   r  layersrY   r   r   r[   s      r\   rN   zJanusVisionEncoder.__init__9  s@     mmeTZTlTlNm$n%<V%D$no$ns   Ar!  rl   s   @r\   r#  r#  8  s    p0 p pr]   r#  c                   $     e Zd Zdef fdZ xZS )JanusVisionModelr   c                 D    t         |   |       t        |      | _        y r  )rM   rN   r#  encoderr  s     r\   rN   zJanusVisionModel.__init__?  s     )&1r]   r!  rl   s   @r\   r*  r*  >  s    20 2 2r]   r*  c                   *     e Zd Zdef fdZd Z xZS )JanusVisionAlignerMLPr   c           	         t         |           t        j                  |j                  |j
                        | _        t        j                  t        d|j                        D cg c],  }t        j                  |j
                  |j
                        . c}      | _
        t        |j                     | _        y c c}w Nrs   )rM   rN   r   r   rC   rS   r  r%  r&  rW   hidden_layersr   rK   r  r(  s      r\   rN   zJanusVisionAlignerMLP.__init__E  s    99V//1F1FG]]NSTUW]WcWcNdeRYYv,,f.C.CDe
 $F$5$56 f   &1B<c                 |    | j                  |      }| j                  D ]  }| j                  |      } ||      } |S r  r  r1  r  rY   r   layers      r\   r   zJanusVisionAlignerMLP.forwardN  G    /'' 	1E ..}=M!-0M	1 r]   )re   rf   rg   r?   rN   r   rk   rl   s   @r\   r.  r.  D  s    70 7r]   r.  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZ	S )JanusVQVAEVectorQuantizerr   c                 N    t         |   |       |j                  gdz  | _        y )Nr*   )rM   rN   ry   quant_state_dimsr  s     r\   rN   z"JanusVQVAEVectorQuantizer.__init__W  s&     !'!3!3 4q 8r]   image_tokensr   c                 B   |j                   d   }| j                  j                  j                   d   }| j                  |      }t        j                  |dd      }|j                  |g| j                  |      }|j                  dddd      j                         }|S )Nr   r   r*   )pdimr   rs   )	r   	embeddingr   F	normalizer   r;  permute
contiguous)rY   r<  r  emb_dimhidden_state_quants        r\   get_codebook_entryz,JanusVQVAEVectorQuantizer.get_codebook_entry[  s    !''*
~~,,2226 "^^L9[[);qbI 044j5b4CXCX5bZa5bc/771aCNNP!!r]   )
re   rf   rg   rn   rN   r   
LongTensorr   rG  rk   rl   s   @r\   r9  r9  V  s/    9/ 9"u/?/? "EDUDU "r]   r9  c                       e Zd Zy)JanusVQVAEResnetBlockNr   rL   r]   r\   rJ  rJ  k      r]   rJ  c                       e Zd Zy)JanusVQVAEAttnBlockNr   rL   r]   r\   rM  rM  o  rK  r]   rM  c                       e Zd Zy)JanusVQVAEConvDownsampleNr   rL   r]   r\   rO  rO  s  rK  r]   rO  c                   $     e Zd Z fdZd Z xZS )JanusVQVAEConvUpsamplec                 t    t         |           t        j                  j	                  ||ddd      | _        y )Nr   rs   kernel_sizestridepadding)rM   rN   r   r   r   conv)rY   rz   r[   s     r\   rN   zJanusVQVAEConvUpsample.__init__x  s.    HHOOK!TU_`Oa	r]   c                 X    t        j                  |dd      }| j                  |      }|S )Ng       @nearest)scale_factormode)rA  interpolaterW  r  s     r\   r   zJanusVQVAEConvUpsample.forward|  s(    m#IV		-0r]   )re   rf   rg   rN   r   rk   rl   s   @r\   rQ  rQ  w  s    br]   rQ  c                   `     e Zd Zdedef fdZdej                  dej                  fdZ xZ	S )JanusVQVAEMidBlockr   channelsc                     t         |           t        |||      | _        t	        |      | _        t        |||      | _        y )Nr   rz   r{   )rM   rN   rJ  block_1rM  attn_1block_2)rY   r   r_  r[   s      r\   rN   zJanusVQVAEMidBlock.__init__  sF    , !

 *(3, !
r]   r   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r  )rb  rc  rd  r  s     r\   r   zJanusVQVAEMidBlock.forward  s2    ]3M2]3r]   )
re   rf   rg   rn   r   rN   r   r   r   rk   rl   s   @r\   r^  r^    s2    
/ 
3 
U\\ ell r]   r^  c                   2    e Zd Zd Zdej
                  fdZy)JanusVQVAEEncoderc           	         t         j                  j                          t        |j                        | _        |j                  | _        |j                  }|j                  }|j                  }|j                  }|j                  }t        j                   j                  ||ddd      | _        dt        |      z   }|| _        t        j                          | _        t%        | j
                        D ]   }t        j                          }	t        j                          }
|||   z  }|||   z  }t%        | j                        D ]N  }|	j'                  t)        |||             |}|| j
                  dz
  k(  s5|
j'                  t+        |             P t        j                         }|	|_        |
|_        || j
                  dz
  k7  rt1        |      |_        | j"                  j'                  |        t5        |      | _        t        j                   j9                  d|dd	      | _        t        j                   j                  ||rd
|z  n|ddd      | _        y )Nr   rs   rS  )rs   ra  rq   ra   T
num_groupsrF   r  affiner*   )r   ModulerN   lenr}   num_resolutionsr~   r|   rz   rw   rx   r   r   conv_intuplein_channel_multiplierr%  downr&  appendrJ  rM  blockattnrO  
downsampler^  midr   norm_outconv_out)rY   r   r|   rz   rw   rx   r}   rq  i_levelrt  ru  block_in	block_outi_blockrr  s                  r\   rN   zJanusVQVAEEncoder.__init__  s   
		"6#<#<=$33,,((,, 00#66xx{MqYZdef $u-?'@ @%:"MMO	T112 	#GMMOE==?D$'<W'EEH%(:7(CCI !4!45 
?)%$,%. %d22Q66KK 3H =>
? 99;DDJDI$..22":8"DIIT"-	#0 &fh7**bxUYbf*g#0Ao ( 
r]   r   c                    | j                  |      g}t        | j                        D ]  }t        | j                        D ]  } | j                  |   j
                  |   |d         }t        | j                  |   j                        dkD  r" | j                  |   j                  |   |      }|j                  |        || j                  dz
  k7  s|j                  | j                  |   j                  |d                 |d   }| j                  |      }| j                  |      }|t        j                  |      z  }| j                  |      }|S )Nr   r   rs   )ro  r&  rn  r~   rr  rt  rm  ru  rs  rv  rw  rx  r   sigmoidry  )rY   r   r   rz  r}  hidden_statelast_hidden_states          r\   r   zJanusVQVAEEncoder.forward  sT   l34T112 		WG !4!45 3@tyy177@!"%  tyy)../!3#C499W#5#:#:7#CL#QL$$\23 $..22$$TYYw%7%B%B=QSCT%UV		W *"- HH%67 !MM*;<U]]+<== MM*;<  r]   N)re   rf   rg   rN   r   rH  r   rL   r]   r\   rg  rg    s    1
f!E$4$4 !r]   rg  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )JanusVQVAEDecoderc           	      v   t         |           t        |j                        | _        |j
                  | _        |j                  }|j                  }|j                  }||j                  | j                  dz
     z  }t        j                  j                  ||ddd      | _        t        ||      | _        t        j                         | _        t#        t%        | j                              D ]  }t        j                         }t        j                         }||j                  |   z  }	t%        | j
                  dz         D ]N  }
|j'                  t)        |||	             |	}|| j                  dz
  k(  s5|j'                  t+        |             P t        j,                         }||_        ||_        |dk7  rt3        |      |_        | j                   j'                  |        t        j                  j7                  d|dd	      | _        t        j                  j                  ||ddd      | _        y )
Nrs   r   rS  ra  r   rq   ra   Tri  )rM   rN   rm  r}   rn  r~   r|   rx   r{   r   r   r   ro  r^  rw  r%  upreversedr&  rs  rJ  rM  rl  rt  ru  rQ  upsampler   rx  ry  )rY   r   r|   rx   r{   r{  rz  rt  ru  r|  r}  r  r[   s               r\   rN   zJanusVQVAEDecoder.__init__  s   "6#<#<=$33,, 00** !6#<#<T=Q=QTU=U#VV xxaXYcde &fh7 --/d&:&: ;< 	GMMOE==?D%(A(A'(JJI !4!4q!89 
?)%$,%. %d22Q66KK 3H =>
? BBHBG!|4X>GGNN2)	. **bxUYbf*g,AVWabcr]   r  r   c                 b   | j                  |      }| j                  |      }t        | j                        D ]  }t        | j                  dz         D ]l  } | j
                  |   j                  |   |      }t        | j
                  |   j                        dkD  sK | j
                  |   j                  |   |      }n || j                  dz
  k7  s| j
                  |   j                  |      } | j                  |      }|t        j                  |      z  }| j                  |      }|S )Nrs   r   )ro  rw  r&  rn  r~   r  rt  rm  ru  r  rx  r   r  ry  )rY   r  rz  r}  s       r\   r   zJanusVQVAEDecoder.forward  s    ||L1 xx- T112 	GG !4!4q!89 P>twww/55g>|Ltwww',,-1#A4777#3#8#8#A,#OLP $..22#www/88F	G }}\2l33}}\2r]   )re   rf   rg   rN   r   r   r   rk   rl   s   @r\   r  r    s)    ,d\E$5$5 %:K:K r]   r  c                        e Zd Zg dZdZdef fdZdej                  dej                  fdZ
eedej                  deej                  ej                  f   fd              Z xZS )	
JanusVQVAE)rM  rJ  r9  r   r   c                 r    t         |   |       t        |      | _        d| _        | j                          y )NF)rM   rN   r  decodergradient_checkpointing	post_initr  s     r\   rN   zJanusVQVAE.__init__1  s0     (0&+# 	r]   r<  r   c                    |j                   d   | j                  j                  d   | j                  j                  d   z  k7  rMt        d| j                  j                  d   | j                  j                  d   z   d|j                    d      | j                  j	                  |      }| j                  |      }| j                  |      }|S )aG  
        Decodes quantized token IDs into pixel values.
        Args:
            image_tokens (torch.LongTensor): Batch of token IDs.
        Returns:
            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
                Pixel values decoded from the token IDs.
        rs   r   z4Expected `image_tokens` to have shape `(batch_size, z)`, but got shape `z`.)r   quantizer;  r   rG  post_quant_convr  )rY   r<  codebook_entryr   r   s        r\   decodezJanusVQVAE.decode9  s     a DMM$B$B1$EHfHfghHi$iiFt}}GeGefgGhkokxkx  lJ  lJ  KL  lM  HM  GN N""."4"4!5R9  99,G,,^<||M2r]   c                     |j                   d   }| j                  |      \  }}}| j                  |j                  |d            }t	        ||      }|S )Nr   r   )r   encoder  r   r   )rY   r   r  quantr   indicesr   r  s           r\   r   zJanusVQVAE.forwardL  sU     "''*
)-\)B&~w#{{7<<
B+GH!"6Gr]   )re   rf   rg   r   main_input_namern   rN   r   rH  r   r  r&   r%   r	   r   rk   rl   s   @r\   r  r  )  s    
 %O/ 5#3#3 8I8I & 	''	 
u  %"3"33	4	  	r]   r  c                   *     e Zd Zdef fdZd Z xZS )JanusVQVAEAlignerMLPr   c           	         t         |           t        j                  |j                  |j
                        | _        t        j                  t        d|j                        D cg c],  }t        j                  |j
                  |j
                        . c}      | _
        t        |j                     | _        y c c}w r0  )rM   rN   r   r   ru   rS   r  r%  r&  rD   r1  r   rK   r  r(  s      r\   rN   zJanusVQVAEAlignerMLP.__init__[  s    99V--v/D/DE]]NSTUW]WoWoNpqRYYv,,f.C.CDq
 $F$5$56 rr2  c                 |    | j                  |      }| j                  D ]  }| j                  |      } ||      } |S r  r4  r5  s      r\   r   zJanusVQVAEAlignerMLP.forwardd  r7  r]   )re   rf   rg   rn   rN   r   rk   rl   s   @r\   r  r  Z  s    7/ 7r]   r  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ	 xZ
S )JanusVQVAEHeadzOHead used for sampling tokens in image generation, replacing the usual lm head.r   c                    t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        t        j                  |j
                  |j                        | _        y r  )rM   rN   r   r   r   rS   proj_outr   rK   r  rv   vision_headr  s     r\   rN   zJanusVQVAEHead.__init__o  s^    		&">">@U@UV#F$5$5699V%:%:F<Q<QRr]   r   r   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r  )r  r  r  r  s     r\   r   zJanusVQVAEHead.forwardu  s6    m4**=9((7r]   )re   rf   rg   rh   rn   rN   r   r   tensorr   rk   rl   s   @r\   r  r  l  s0    YS/ SU\\ ell r]   r  zl
    The Janus model which consists of a siglip vision backbone, a Llama language model and a VQ model.
    )custom_introc                   \    e Zd Zdef fdZd Zd Zd Zee		 	 	 	 	 	 	 	 	 	 	 dde
j                  de
j                  dee
j                     d	ee
j                     d
ee   dee
j                     dee
j                     dee   dee   dee   deee
j                  f   fd              Z xZS )
JanusModelr   c                    t         |   |       || _        t        j	                  |j
                        | _        t        | j                  j                        | _        t        j	                  |j                        | _        t        j                  | j                  j                  j                  | j                  j                  j                        | _        t#        | j                  j                        | _        t'        | j                  j                        | _        t+        j,                  |j.                        | _        d| _        | j5                          y )N)r   F)rM   rN   r   r*  _from_configrA   vision_modelr.  alignerr  r   vqmodelr   r   rv   ru   generation_embeddingsr  generation_alignerr  generation_headr+   from_configr   language_modelr  r  r  s     r\   rN   zJanusModel.__init__  s     ,99&:N:NO,T->->-E-EF!..v/?/?@ &(\\$,,2E2E2T2TVZVbVbViViVsVs%t""6t||7J7J"K-dll.A.AB'336;M;MN&+#r]   c                 6    | j                   j                         S r  )r  get_input_embeddingsrY   s    r\   r  zJanusModel.get_input_embeddings  s    ""7799r]   c                 :    | j                   j                  |       y r  )r  set_input_embeddingsrY   values     r\   r  zJanusModel.set_input_embeddings  s    007r]   c                 ^    | j                  |      }| j                  |j                        }|S r  )r  r  r  )rY   r   image_embedss      r\   get_image_featureszJanusModel.get_image_features  s,    ((6||L$B$BCr]   	input_idsr   r   r   r   cache_positioninputs_embeds	use_cacher   output_hidden_stateslogits_to_keepc                 D   |	|	n| j                   j                  }	|
|
n| j                   j                  }
|d u |d uz  rt        d      | j                  r%| j
                  r|rt        j                  d       d}||t        d      | | j                         |      }|| j                  |      }|| j                   j                  k(  }|j                  d   }|j                  d|      }|j                  d      j                  dd|      }|j                  |j                   |j"                        }|j%                  ||      } | j&                  d||||||	|
||d	|}t)        |j*                  |j,                  |j.                  |j0                  |nd       }|S )	NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either onezZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FzdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either oner   )	r  r   r   r   r  r   r  r  r  )r  r   r   
attentionsimage_hidden_statesrL   )r   r   r  r   r  r  r   r  r  r  r   r   r   	unsqueezeexpandr   devicer   masked_scatterr  r   r  r   r   r  )rY   r  r   r   r   r   r  r  r  r   r  r  rZ   r  image_attention_maskru   image_features	lm_outputr  s                      r\   r   zJanusModel.forward  s   " 2C1N-TXT_T_TqTq$8$D $++JjJj 	 -t";<s  &&4==##p "	#(Av   7D557	BM#22<@L#,0J0J#J %++B/I)11"i@N#7#A#A"#E#L#LRQSU^#_ +..}/C/C]EXEXYN)889M~^M'D'' 
')%+/!5))
 
	 .'99%55#11 ++0<0Hd
 r]   )NNNNNNNNNNr   )re   rf   rg   r   rN   r  r  r  r&   r%   r   rH  r   r   r   r   r   r
   r   r   rk   rl   s   @r\   r  r  |  s*   { *:8
  '+*.1537+/5959$(,0/334H##H ''H !.	H
 u//0H "%H !!1!12H   1 12H D>H $D>H 'tnH c5<</0H  Hr]   r  c                   |    e Zd ZddgZdZdef fdZd Zd Zde	j                  d	e	j                  fd
Zd Zd Zd Zd Zee	 	 	 	 	 	 	 	 	 	 	 	 d!de	j$                  de	j&                  dee	j                     dee	j$                     dee   dee	j$                     dee	j&                     dee	j$                     dee   dee   dee   deee	j                  f   fd              Z	 	 	 	 	 	 d" fd	Zde	j                  fdZe	j8                  	 	 	 d#de	j                  dee	j$                     dee   f fd        Z xZS )$JanusForConditionalGenerationz(model.language_model.embed_tokens.weightzlm_head.weightTr   c                     t         |   |       || _        t        |      | _        t        j                  |j                  j                  |j                  j                  d      | _
        | j                          y )NFr   )rM   rN   r   r  r   r   r   r   rC   
vocab_sizelm_headr  r  s     r\   rN   z&JanusForConditionalGeneration.__init__  s\     '
yy!3!3!?!?ASASA^A^ejk 	r]   c                 J    | j                   j                  j                         S r  )r   r  r  r  s    r\   r  z2JanusForConditionalGeneration.get_input_embeddings  s    zz((==??r]   c                 N    | j                   j                  j                  |       y r  )r   r  r  r  s     r\   r  z2JanusForConditionalGeneration.set_input_embeddings  s    

!!66u=r]   inputsr   c                 r    | j                   j                  |      }| j                   j                  |      }|S r  )r   r  r  )rY   r  r  s      r\   'prepare_embeddings_for_image_generationzEJanusForConditionalGeneration.prepare_embeddings_for_image_generation  s0    zz77?zz44\Br]   c                     | j                   S r  r  r  s    r\   get_output_embeddingsz3JanusForConditionalGeneration.get_output_embeddings  s    ||r]   c                     || _         y r  r  )rY   new_embeddingss     r\   set_output_embeddingsz3JanusForConditionalGeneration.set_output_embeddings
  s	    %r]   c                     || _         y r  r   )rY   r  s     r\   set_decoderz)JanusForConditionalGeneration.set_decoder  s	    
r]   c                     | j                   S r  r  r  s    r\   get_decoderz)JanusForConditionalGeneration.get_decoder  s    zzr]   r  r   r   r   r   r  r  labelsr  r   r  r  c                    |
|
n| j                   j                  }
||n| j                   j                  } | j                  d|||||||	|
||d
|}|j                  }t        |t              rt        | d      n|}| j                  |dd|ddf         }d}|2| j                  ||| j                   j                  j                        }t        |||j                  |j                  |j                  |j                         }|S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)
r  r   r   r   r   r  r  r   r  r  )logitsr  r  )lossr  r   r   r  r  rL   )r   r   r  r   r  r   r   slicer  loss_functionr   r  r   r   r   r  r  )rY   r  r   r   r   r   r  r  r  r  r   r  r  rZ   r  r   slice_indicesr  r  r  s                       r\   r   z%JanusForConditionalGeneration.forward  s(   0 2C1N-TXT_T_TqTq$8$D $++JjJj 	 $** 
%)%+'/!5)
 
  118B>SV8W~ot4]kmA}a,?@A%%VFt{{OfOfOqOq%rD,#33!//)) ' ; ;
 r]   c           	      N    t        
|   |f|||||d|}	|d   dk(  r||	d<   |	S )N)r   r  r   r  r  r   r   )rM   prepare_inputs_for_generation)rY   r  r   r   r   r  r  r  rZ   model_inputsr[   s             r\   r  z;JanusForConditionalGeneration.prepare_inputs_for_generationP  sT     w<
+')))
 
 !!+7L(r]   r<  c                 x    | j                   j                  j                  |      }|j                  dddd      }|S )a,  
        Decodes generated image tokens from language model to continuous pixel values
        with VQGAN module via upsampling.
        Args:
            image_tokens (`torch.LongTensor` of shape `(batch_size, num_of_tokens)`):
                The tensors corresponding to the input images.
        r   r*   r   rs   )r   r  r  rC  )rY   r<  decoded_images      r\   decode_image_tokensz1JanusForConditionalGeneration.decode_image_tokensn  s:     

**11,?%--aAq9r]   logits_processorc           	         |j                  d| j                        }t        j                  |      }|j                  dd      }|dk(  rt	        %|   d|||d d|S  |j                  di |}|j                         t        j                  t        j                  fvrt        d      |j                          | j                  |j                                ||n	t               }d|d<   |j                  t         j#                  d       d	|_        |j                  |d
<   | j%                  ||j&                  |      \  }}	}|j(                  |j*                  }}
t-        |j.                        dk7  rt        d|j.                   d      |d u}| j1                  |||j*                         |j                  r:|j                  dkD  r+|j3                  t5        |j                               d |_        | j7                  ||j.                  d   |d ||      } | j8                  d|||j:                  d|\  }}| j<                  j>                  j@                  jB                  }|j.                  \  }}|jE                  dd      }|j                  dd       }|jE                  dd      }||d<   ||d d d f   |j&                  k7  ||d d d f   |jF                  d   k7  z  }||d d d f   jI                  ||jJ                          | jM                         |      }| jO                  |||      }|jQ                  dd       A| jS                  |jT                  xs d|dz  tW        |jX                  ||z         ||      |d<   t[        j\                  ||f|
|      }|j^                  }|j`                  }|jb                  }|jd                  }|jf                  }|r|rdnd }|r|rdnd }|r|rdnd }|r|rdnd }ti        |      D ]x  } | jj                  d||d|}|d   jm                  |j*                        |d<   |d   jm                  |j*                        |d<    | j<                  jn                  di |||d}| jq                  ||      }|jr                  d d dd d f   ju                         } | j<                  jw                  |       }! |||!      }"|jx                  r>t[        jz                  |"d      }#t[        j|                  |#d      j                  d      }$nt[        j                  |"d      }$|$|d d |f<   t[        j                  |$|$g      }$|$j                  d      }$| j                  |$      }{ |r@|r|!fz  }|r| j                         fz  }|r|j                  z  }|r|j                  z  }|rt        |!|||j                        S |S ) Ngeneration_configgeneration_modetext)r  r   r  guidance_scalezGot incompatible mode for Image Generation, should be one of greedy or sampling. Ensure that beam search is de-activated by setting `num_beams=1` and `num_beam_groups=1`.Tr  zU`guidance_scale` is required for CFG but not provided. Setting to default value of 5.   r  r*   z;Expected input ids of shape (batch_size, seq_len), but got z3Passing `inputs embeds` is not supported currently.)r  rs   )r  input_ids_seq_lengthencoder_input_idsprefix_allowed_tokens_fnr  r  )r  r   expand_sizer   boi_token_idr   static)cache_implementationr  max_cache_lenr  model_kwargs)r   r  rL   )r  r  r  )r   r  r   )r?  )num_samples)	sequencesscoresr  r  r   r   )Ipopr  copydeepcopyrM   generateupdateget_generation_moder   SAMPLEGREEDY_SEARCHr   validate_validate_model_kwargsr   r  r   warning_prepare_model_inputsbos_token_idr   r  rm  r   _prepare_special_tokensrs  r   _get_logits_processor_expand_inputs_for_generationnum_return_sequencesr   r  r   rX   repeatgeneration_kwargsmasked_fill_pad_token_idr  _get_initial_cache_positionr   
_get_cacher  max
max_lengthr   zerosr   r  output_scoresoutput_logitsreturn_dict_in_generater&  r  r   r  #_update_model_kwargs_for_generationr  cloner  	do_samplesoftmaxmultinomialsqueezeargmaxcatr  r  r   r  r   r   r   )&rY   r  r   r  rZ   r  r  r  r  model_input_namer   r  kwargs_has_attention_maskrX   r  r  input_tokensmaskr  generated_tokensr   r  r  r  r  
raw_scores
raw_logitsdecoder_hidden_statesdecoder_attentionsir  r  r  r  next_token_scoresprobs
next_tokenr[   s&                                        r\   r  z&JanusForConditionalGeneration.generatez  s    #JJ':D<R<RS MM*;< !**%6?f$7# -"3#	
   0(//9&9 002>;P;PR`RnRn:ool  	""$##L$5$5$78 0@/K+QdQf %)[!++3NNrs/0,):)I)I%& 594N4N%22L5
1	#\ ")9)9vy1$MiooM^EF  %3$$>!$$%68QZcZjZj$k ++0A0P0PST0T##$IJ[JjJj$kl/3,  55/!*!3'%)- 6 
 #E$"D"D #
))>>#
 	#
	<  ::2299JJ'oo
G ''1-%))*:DA'..q!4)7%& Z[!^,0A0N0NNa(,=,O,OP^,__
 	Z[!^$11$8I8V8VW3113LA77V-t4<.2oo%6%K%K%Wx%>!"3">">@PSZ@Z[) /> /L*+ !;;
4D'EU[ab .??0EE)77)77"3"K"K3RD
3RD
'>CW^b$;@QRX\'( #	UA=4== +|GSL .::J-K-N-N}OcOc-dL)*-9:J-K-N-N}OcOc-dL)*/djj// "3%9G  CCG\ZL"44QAX>DDFL ZZ//=F 0F C !**&7R@"..u!DLLRP
"\\*;D
%/QT" J
#;<J#--b1J HHTMG#	UJ #vi'
|11355
 "g&8&88"#%)>)>>%",*!-3 ' 7 7  $#r]   )NNNNNNNNNNNr   )NNNNNN)NNN) re   rf   rg   _tied_weights_keysr   r   rN   r  r  r   r   r  r  r  r  r  r&   r%   rH  r   r   r   r   r
   r   r   r  r  no_gradr   r  rk   rl   s   @r\   r  r    s   DFVW!{ @>ell u|| 
&  '+*.1537+/5959-1$(,0/3349##9 ''9 !.	9
 u//09 "%9 !!1!129   1 129 ))*9 D>9 $D>9 'tn9 c5<</09  9| <
 
 ]]  $59:>	}$}$ !!1!12}$ ##67	}$ }$r]   r  c                       e Zd ZdZdddej
                  ddddddf
dedeee	e
f      de
d	ed
edee
ef   dedeeeee   f      deeeee   f      dee   f fdZ	 	 	 ddej                   dee
ee
e
e
f   f   deee	ef      deee	ef      dej&                  f
dZej
                  ddfdej                   deee	e
f   e
f   d	edeee	ef      deee	ef      dej                   fdZ	 	 	 	 	 	 	 dded
ee   dee   dee   deee      deee      dee	   dee	   fdZ	 ddej&                  deeee   f   deeee   f   deee	ef      dej&                  f
dZ xZS )JanusImageProcessora
  
    Constructs a JANUS image processor.

    Args:
        do_resize (`bool`, *optional*, defaults to `True`):
            Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the
            `do_resize` parameter in the `preprocess` method.
        size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`):
            Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess`
            method.
        min_size (`int`, *optional*, defaults to 14):
            The minimum allowed size for the resized image. Ensures that neither the height nor width
            falls below this value after resizing.
        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
            Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be
            overridden by the `resample` parameter in the `preprocess` method.
        do_rescale (`bool`, *optional*, defaults to `True`):
            Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the
            `do_rescale` parameter in the `preprocess` method.
        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
            Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be
            overridden by the `rescale_factor` parameter in the `preprocess` method.
        do_normalize (`bool`, *optional*, defaults to `True`):
            Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess`
            method. Can be overridden by the `do_normalize` parameter in the `preprocess` method.
        image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`):
            Mean to use if normalizing the image. This is a float or list of floats the length of the number of
            channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be
            overridden by the `image_mean` parameter in the `preprocess` method.
        image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`):
            Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
            number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
            Can be overridden by the `image_std` parameter in the `preprocess` method.
        do_convert_rgb (`bool`, *optional*, defaults to `True`):
            Whether to convert the image to RGB.
    TN   gp?	do_resizer   min_sizeresample
do_rescalerescale_factordo_normalize
image_mean	image_stddo_convert_rgbc           	          t        |   di | || _        |d| _        y t	        |D cg c]  }t        |dz         c}      | _        y c c}w )N)   rA  rA     rL   )rM   rN   r8  background_colorrp  r   )rY   r7  r   r8  r9  r:  r;  r<  r=  r>  r?  rZ   xr[   s                r\   rN   zJanusImageProcessor.__init__a  sM     	"6" $3D!$)*LA3q3w<*L$MD!*Ls   AimagerC  data_formatinput_data_formatr   c                 N   t        ||      \  }}|t        j                  k(  r|j                  d   n|j                  d   }||k(  r|t	        |||      }|S |}|S t        ||      }t        |t              r|g}nt        |      |k7  rt        d| d      |t        j                  k(  r~t        j                  |||f|j                        }	t        |      D ]  \  }
}||	|
ddddf<    ||kD  r||z
  dz  }||	dd|||z   ddf<   |	S ||z
  dz  }||	dddd|||z   f<   |	S t        j                  |||f|j                        }	t        |      D ]  \  }
}||	dddd|
f<    ||kD  r||z
  dz  }||	|||z   ddddf<   |	S ||z
  dz  }||	dd|||z   ddf<   |	S )a}  
        Pads an image to a square based on the longest edge.

        Args:
            image (`np.ndarray`):
                The image to pad.
            background_color (`int` or `Tuple[int, int, int]`, *optional*, defaults to 0):
                The color to use for the padding. Can be an integer for single channel or a
                tuple of integers representing for multi-channel images. If passed as integer
                in mutli-channel mode, it will default to `0` in subsequent channels.
            data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format for the output image. Can be one of:
                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                If unset, will use same as the input image.
            input_data_format (`str` or `ChannelDimension`, *optional*):
                The channel dimension format for the input image. Can be one of:
                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.

        Returns:
            `np.ndarray`: The padded image.
        r   r   Nz(background_color must have no more than z) elements to match the number of channelsr   r*   )r   r   FIRSTr   r   r  r   r   rm  r   npr  r   	enumerate)rY   rE  rC  rF  rG  r   r   rF   max_dimresultr.  colorstarts                r\   pad_to_squarez!JanusImageProcessor.pad_to_squarew  s+   < 'u.?@):>N>T>T)Tu{{1~Z_ZeZefhZiU? * ,E;@QR 
 L  
 Lfe$ &, 01!"l2:<.Hqr   0 6 66XX|Wg>ekkRF%&67 (5"'q!Qw(v~ 6)a/7<q%%&.0!34  !5Q.6;q!UUU]223  XXw>ekkRF%&67 (5"'q!Qw(v~ 6)a/7<uuv~-q!34
  !5Q.6;q%%%-/23r]   c                    |t        |      }t        ||      \  }}t        ||      }	t        |d      }|d   |d   k7  rt	        d|d    d|d          |d   }||	z  }
t        t        ||
z        | j                        t        t        ||
z        | j                        g}t        |f||||d|}| j                  || j                  |      }|S )	a  
        Resize an image to dynamically calculated size.

        Args:
            image (`np.ndarray`):
                Image to resize.
            resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):
                `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`.
            data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the output image. If unset, the channel dimension format of the input
                image is used. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `None`: will be inferred from input
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.

        Returns:
            `np.ndarray`: The resized image.
        T)default_to_squarer   r   z5Output height and width must be the same. Got height=z and width=)r   r9  rF  rG  )rE  rC  rG  )
r   r   r  r   r   r   r8  r   rP  rC  )rY   rE  r   r9  rF  rG  rZ   r   r   max_sizedeltaoutput_size_nonpaddeds               r\   r   zJanusImageProcessor.resize  s   B $ >u E&u.?@vu%TT:>T']*GXGWWbcghocpbqr  H~x FUN#T]]3EEM"DMM2!

 
&#/
 
 ""!22/ # 

 r]   imagesreturn_tensorsc	                    ||n| j                   }|d| j                  z  n|}||n| j                  }||n| j                  }||n| j                  }t        |      }t        |d   t        j                  j                        rt        |      dkD  r|S |d   S |t        |d         }g }	|D ]  }
t        |
      }
|r| j                  |
|||      }
|rC| j                  |
||      }
|
j                  dd      j                  t         j"                        }
|rB|r@|dk(  r;t%        |
t&        j(                  |	      }
t        j                  j+                  |
      }
|	j-                  |
        d
|	i}|dk7  r|nd}t/        ||      S )znApplies post-processing to the decoded image tokens by reversing transformations applied during preprocessing.Nr   r   rs   )rE  r=  r>  rG  )r   rG  rB  zPIL.Image.Image)input_channel_dimr   )r   tensor_type)r:  r;  r<  r=  r>  r   r   PILImagerm  r   r   unnormalizerescaleclipastyperJ  uint8r   r   LAST	fromarrayrs  r   )rY   rV  r:  r;  r<  r=  r>  rG  rW  r   rE  r   s               r\   postprocesszJanusImageProcessor.postprocess  s    $.#9Zt
6D6Lt222R`'3'?|TEVEV#-#9Zt
!*!6IDNN	$V,fQi1 [1_6;&);$ >vay I 	'E"5)E((J)_p )  U.Tef

1c*11"((;
~AR/R3E;K;P;Pduv		++E2&!	'$ -+9=N+NTX>BBr]   c                    d}t        |t              r(t        |      |k7  r t        d| dt        |             |g|z  }t        |t              r(t        |      |k7  r t        d| dt        |             |g|z  }t	        d t        ||      D              }t	        d |D              }| j                  ||||      }|S )a~  
        Unnormalizes `image` using the mean and standard deviation specified by `mean` and `std`.
        image = (image * image_std) + image_mean
        Args:
            image (`torch.Tensor` of shape `(batch_size, num_channels, image_size, image_size)` or `(num_channels, image_size, image_size)`):
                Batch of pixel values to postprocess.
            image_mean (`float` or `Iterable[float]`):
                The mean to use for unnormalization.
            image_std (`float` or `Iterable[float]`):
                The standard deviation to use for unnormalization.
            input_data_format (`ChannelDimension` or `str`, *optional*):
                The channel dimension format for the input image. If unset, the channel dimension format is inferred
                from the input image. Can be one of:
                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
        r   zmean must have z$ elements if it is an iterable, got zstd must have c              3   .   K   | ]  \  }}| |z    y wr  rL   ).0r   r   s      r\   	<genexpr>z2JanusImageProcessor.unnormalize.<locals>.<genexpr>_  s     WytSus{Ws   c              3   &   K   | ]	  }d |z    yw)rs   NrL   )rg  r   s     r\   rh  z2JanusImageProcessor.unnormalize.<locals>.<genexpr>`  s     ;#a#g;s   )rE  r   r   rG  )r   r   rm  r   rp  ziprB  )rY   rE  r=  r>  rG  rF   rev_image_meanrev_image_stds           r\   r]  zJanusImageProcessor.unnormalize9  s    0 j(+:,. ?<.@dehisetdu!vww$4Ji*9~- >,?cdghqdrcs!tuu"l2IWC
I<VWW;;;n-Sd  
 r]   )r   NN)NNNNNNNr  )re   rf   rg   rh   r   BICUBICr   r   r   strr   r
   r   r   rN   rJ  ndarrayr	   r   arrayrP  r   r   rd  r   r]  rk   rl   s   @r\   r5  r5  ;  s   #N )-'9'A'A,3!:>9=)-NN tCH~&N 	N
 %N N c5j)N N U5$u+#567N E%e"456N !N2 >?>BDHHzzH  U3S=%9 9:H eC)9$9:;	H
 $E#/?*?$@AH 
H\ (:'A'A>BDHCzzC DcNC'(C %	C
 eC)9$9:;C $E#/?*?$@AC 
CP &**.'+,0+/+/(,1C1C TN1C !	1C
 tn1C T%[)1C DK(1C $C=1C !1Cp EI+xx+ %%01+ /0	+
 $E#/?*?$@A+ 
+r]   r5  )	r5  r   r  r  r  r*  rn   r?   r   )|r  dataclassesr   typingr   r   r   r   r   r	   r
   numpyrJ  r   r   .transformers.models.blip.image_processing_blipr   activationsr   cache_utilsr   
generationr   r   r   r   generation.utilsr   image_processing_utilsr   r   image_transformsr   r   image_utilsr   r   r   r   r   r   r   modeling_flash_attention_utilsr    modeling_outputsr!   modeling_utilsr"   r#   processing_utilsr$   utilsr%   r&   r'   r(   r)   autor+   blip_2.modeling_blip_2r,   !chameleon.configuration_chameleonr-   chameleon.modeling_chameleonr.   r/   r0   r1   r2   r3   idefics.modeling_ideficsr4   r5   llama.modeling_llamar6   siglip.configuration_siglipr7   siglip.modeling_siglipr8   r9   r:   torch.nntorch.nn.functional
functionalrA  torch.utils.checkpointr[  configuration_utilsr;   r<   r=   
get_loggerre   r   r?   rn   r   r   r   r   r   r   rl  r   r  r  r#  r*  r.  r9  rJ  rM  rO  rQ  r^  rg  r  r  r  r  r  r  r5  __all__rL   r]   r\   <module>r     s     ! I I I    M !   u u 9 A C   C + F & g g  5 D  e : < ^ ^ ##! 3 - 
		H	%
^1* ^1BW+ Wtj#" j#Z ?? ? ?@ -{ - - 	#A 	 	 	"? 	 	2 "R299 RjRYY (*0 *p p2' 2BII $" = "*	< 		8 		B 	RYY  ,J!-ryy J!ZA		 AH. .b299 $RYY   
k% k
k\I$$8/ I$X
i, iX	
r]   