UhWdZddlZddlZddlmZddlmZmZmZm Z m Z ddl Z ddl Z ddl m Z ddlmZmZmZddlmZmZdd lmZdd lmZmZmZmZmZmZmZmZdd l m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)ddl*m+Z+e)jXe-Z.dEdZ/Gdde j`Z1Gdde j`Z2Gdde j`Z3de2iZ4Gdde j`Z5Gdde j`Z6Gdde j`Z7Gdde j`Z8Gd d!e j`Z9Gd"d#e j`Z:Gd$d%e j`Z;e(Gd&d'e!ZGd,d-e j`Z?Gd.d/e j`Z@e(d01Gd2d3e<ZAe(d41Gd5d6e<ZBe(d71Gd8d9e<ZCe(d:1Gd;de<ZEe(Gd?d@e<ZFe(dA1GdBdCe| |_ |S#t$rtj dwxYw#t6$r1}|xj8|j4| j4fz c_d}~wwxYw#tB$r}t;d|| |Yd}~d}~wwxYw)&z'Load tf checkpoints in a pytorch model.rNzLoading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape zelectra/embeddings/zgenerator/embeddings/ generatorzelectra/zdiscriminator/z generator/dense_1dense_predictionz!generator_predictions/output_biaszgenerator_lm_head/bias/c3$K|]}|dv yw)) global_step temperatureN).0ns ~/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/electra/modeling_electra.py z-load_tf_weights_in_electra..^sE1166Ez Skipping z [A-Za-z]+_\d+z_(\d+)kernelgammaweight output_biasbetabiasoutput_weightssquad classifierr _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )"renumpy tensorflow ImportErrorloggererrorospathabspathinfotrainlist_variables load_variableappendzip isinstanceElectraForMaskedLMreplacesplitany fullmatchgetattrlenintendswith transposeshape ValueErrorargsprinttorch from_numpydataAttributeError)modelconfigtf_checkpoint_pathdiscriminator_or_generatorr;nptftf_path init_varsnamesarraysnamerUarray original_namepointerm_name scope_namesnumes r-load_tf_weights_in_electraro4sz ggoo01G KK8 BC''0I E F  e (l5'BC&&w5 T e  5&)6 e! 3 %!34||$9;RS)[8||J0@A||L*=<< +=>D<< CE]^D::c?DEEE i 78G +<< 0&9"$((9f"=K#)(Kq>X-Q71J%gx8G ^}4 A&8P%gv6G ^'77%gx8G ^w.%g|G{#q(k!n-C%clG# +$}-!'848# U+ ==EKK/$~gmm_DUV[VaVaUbbm%noo0 .tf5} E ++E2GLg6n LQ   Q   @ 7==%++66    Im_-tQ 7  sV K1B*MB0M9A M?L*M1 L M,M  MM M5M00M5c eZdZdZfdZ d deejdeejdeejdeejde dejf d Z xZ S) ElectraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c>t|tj|j|j |j |_tj|j|j |_ tj|j|j |_ tj|j |j|_ tj|j|_|j#dt%j&|jj)ddt+|dd|_|j#d t%j.|j0j3t$j4 dy) N) padding_idxeps position_ids)r F) persistentposition_embedding_typeabsolutetoken_type_idsdtype)super__init__r Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_bufferrYarangeexpandrPryzerosrvsizelongselfr^ __class__s r-rzElectraEmbeddings.__init__s1 !||F,=,=v?T?Tbhbubuv#%<<0N0NPVPePe#f %'\\&2H2H&J_J_%`"f&;&;AVAVWzz&"<"<=   ELL)G)GHOOPWXej  (/v7PR\']$  ekk$*;*;*@*@*B%**Ubg  input_idsr{rv inputs_embedspast_key_values_lengthreturncZ||j}n|jdd}|d}||jdd|||zf}|st|dr-|jddd|f}|j |d|} | }n:t j |t j|jj}||j|}|j|} || z} |jdk(r|j|} | | z } |j| } |j| } | S)Nrwr r{rr}devicerz)rrvhasattrr{rrYrrrrrryrrr) rrr{rvrr input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr embeddingsrs r-forwardzElectraEmbeddings.forwardsH  #..*K',,.s3K ^  ,,Q0FVlIl0l-lmL  !t-.*.*=*=a*n*M'3J3Q3QR]^_R`bl3m0!A!&[ SWSdSdSkSk!l   00;M $ : :> J"%::  ' ': 5"&":":<"H  - -J^^J/ \\*- r)NNNNr) __name__ __module__ __qualname____doc__rrrY LongTensor FloatTensorrRTensorr __classcell__rs@r-rqrqsQ .15593759&' 'E,,-'!!1!12'u//0 '   1 12 ' !$ ' 'rrqcPeZdZd fd Zdej dej fdZ ddej deejdeejdeejd eejd ee e ejd ee de ej fd Z xZ S)ElectraSelfAttentionct||j|jzdk7r2t |ds&t d|jd|jd|j|_t |j|jz |_|j|jz|_tj|j|j|_ tj|j|j|_ tj|j|j|_ tj|j|_|xs t#|dd|_|j$dk(s|j$d k(rF|j&|_tj(d |j&zd z |j|_|j,|_y) NrrzThe hidden size (z6) is not a multiple of the number of attention heads ()ryrz relative_keyrelative_key_queryr9r )r~r hidden_sizenum_attention_headsrrVrRattention_head_size all_head_sizer Linearquerykeyvaluerattention_probs_dropout_probrrPryrrdistance_embedding is_decoderrr^ryrs r-rzElectraSelfAttention.__init__s     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F( ' -zC $  ' '> 9T=Y=Y]q=q+1+I+ID (&(ll1v7U7U3UXY3Y[_[s[s&tD # ++rxrc|jdd|j|jfz}|j|}|j ddddS)Nrwrr9r r )rrrviewpermute)rr new_x_shapes r-transpose_for_scoresz)ElectraSelfAttention.transpose_for_scoressLffhsmt'?'?AYAY&ZZ FF; yyAq!$$r hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsc$|j|}|du} | r||d} |d} |}n| rC|j|j|} |j|j|} |}n|y|j|j|} |j|j|} t j |d| gd} t j |d| gd} n@|j|j|} |j|j|} |j|} |du} |j r| | f}t j| | jdd}|jdk(s|jdk(r| jd| jd}}| rDt j|dz tj|j jdd}n@t j|tj|j jdd}t j|tj|j jdd}||z }|j!||j"zdz }|j%| j& }|jdk(rt j(d | |}||z}nE|jdk(r6t j(d | |}t j(d | |}||z|z}|t+j,|j.z }|||z}t0j2j5|d}|j7|}|||z}t j|| }|j9dddd j;}|j=dd|j>fz}|j|}|r||fn|f}|j r||fz}|S)Nrr r9dimrwrrrr|zbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rrrrrYcatrmatmulrTryrUtensorrrrrrrtor}einsummathsqrtrr functionalsoftmaxrr contiguousrr)rrrrrrrrmixed_query_layeris_cross_attention key_layer value_layer query_layer use_cacheattention_scores query_length key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapeoutputss r-rzElectraSelfAttention.forwards!JJ}5 3$> ."<&q)I(+K3N 11$((;P2QRI33DJJ?T4UVK3N  '11$((=2IJI33DJJ}4MNK >!#4i"@aHI))^A%6 $D!LK11$((=2IJI33DJJ}4MNK//0AB "$. ??(5N!<< Y5H5HR5PQ  ' '> 9T=Y=Y]q=q'2'8'8';Y__Q=O*L!&j1nEJJWdWkWk!l!q!q""'l%**UbUiUi!j!o!oprtu!v"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||S"S%**+BC 6G=/2mM] ?? 11GrNNNNNNF) rrrrrYrrrrrboolrrrs@r-rrs,4%ell%u||%7;15=A>BDH,1c||c!!2!23cE--. c ((9(9: c !)):): ; c!uU->->'?!@Ac$D>c u|| crrcneZdZfdZdej dej dej fdZxZS)ElectraSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNrt) r~rr rrdenserrrrrrs r-rzElectraSelfOutput.__init__Qs` YYv1163E3EF f&8&8f>S>STzz&"<"<= rr input_tensorrcr|j|}|j|}|j||z}|Srrrrrrrs r-rzElectraSelfOutput.forwardW7 =1  ]3 }|'CD rrrrrrYrrrrs@r-rrP1> U\\RWR^R^rreagerceZdZd fd ZdZ d dej deejdeejdeejdeejdee e ejd ee d e ej fd Z xZ S)ElectraAttentionct|t|j|||_t ||_t|_y)Nry) r~rELECTRA_SELF_ATTENTION_CLASSES_attn_implementationrroutputset pruned_headsrs r-rzElectraAttention.__init__esC 263N3NO ,C  (/ Erc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrr r)rQrrrrrrrrrrrrunion)rheadsindexs r- prune_headszElectraAttention.prune_headsms u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:rrrrrrrrrc p|j|||||||}|j|d|} | f|ddz} | S)Nrr )rr) rrrrrrrr self_outputsattention_outputrs r-rzElectraAttention.forwardsWyy    ! "    ;;|A F#% QR(88rrr) rrrrr rYrrrrrrrrs@r-rrds";*7;15=A>BDH,1||!!2!23E--.  ((9(9:  !)):): ; !uU->->'?!@A$D> u|| rrcVeZdZfdZdej dej fdZxZS)ElectraIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r~rr rrintermediate_sizerrJ hidden_actstrrintermediate_act_fnrs r-rzElectraIntermediate.__init__s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rrrcJ|j|}|j|}|Sr)rr)rrs r-rzElectraIntermediate.forwards& =1 00? rrrs@r-rrs#9U\\ellrrcneZdZfdZdej dej dej fdZxZS) ElectraOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r~rr rrrrrrrrrrs r-rzElectraOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= rrrrcr|j|}|j|}|j||z}|Srrrs r-rzElectraOutput.forwardrrrrs@r-rrrrrceZdZfdZ d dej deejdeejdeejdeejdeeeejdee d eej fd Z d Z xZ S) ElectraLayercft||j|_d|_t ||_|j |_|j|_|jr,|j st|dt |d|_ t||_ t||_ y)Nr z> should be used as a decoder model if cross attention is addedrzr)r~rchunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionrVcrossattentionr intermediaterrrs r-rzElectraLayer.__init__s '-'E'E$)&1 ++#)#=#=  # #?? D6)g!hii"26S]"^D /7#F+ rrrrrrrrrc ||ddnd}|j|||||} | d} |jr | dd} | d} n| dd} d} |jrT|Rt|dstd|d||d dnd}|j | ||||||}|d} | |ddz} |d}  | z} t |j |j|j| }|f| z} |jr|  fz} | S) Nr9)rrrr rwr$z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) r"rrrVr$rfeed_forward_chunkr r!)rrrrrrrrself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputs layer_outputs r-rzElectraLayer.forwards}:H9S>"1#5Y] !%   /3 "0" 2!4 ??,Qr2G 6r : ,QR0G'+$ ??4@4!12 =dVDDD @N?Yrs(;_c %&*&9&9 %&)!' # 7q9  7" ==G,C2+F ( 14P P 0  # #T%A%A4CSCSUe  /G+ ??!2 44GrcL|j|}|j||}|Sr)r%r)rrintermediate_outputr.s r-r'zElectraLayer.feed_forward_chunks,"//0@A{{#68HI rr) rrrrrYrrrrrrr'rrs@r-rrs ,"7;15=A>BDH,1?||?!!2!23?E--. ? ((9(9: ? !)):): ; ?!uU->->'?!@A?$D>? u|| ?BrrcDeZdZfdZ ddej deejdeejdeejdeejdeeeejdee d ee d ee d ee d e eej e ffd Z xZ S)ElectraEncoderct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}w)NF) r~rr^r ModuleListrangenum_hidden_layersrlayergradient_checkpointing)rr^_rs r-rzElectraEncoder.__init__sN  ]]%H`H`Ba#bQL$8#bc &+#$csA#rrrrrpast_key_valuesrroutput_hidden_states return_dictrc | rdnd} |rdnd} |r|jjrdnd} |jr%|jr|rtj dd}|rdnd}t |jD]\}}| r| |fz} |||nd}|||nd}|jr/|jr#|j|j|||||||}n||||||||}|d}|r ||dfz }|s|| |dfz} |jjs| |dfz} | r| |fz} | std||| | | fDSt||| | | S) Nr*zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frrwr r9c3$K|]}|| ywrr*)r+vs r-r.z)ElectraEncoder.forward..Ws = r/)last_hidden_stater:r attentionscross_attentions) r^r#r8trainingr? warning_once enumerater7_gradient_checkpointing_func__call__tupler)rrrrrrr:rrr;r<all_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cachei layer_modulelayer_head_maskr layer_outputss r-rzElectraEncoder.forwards#7BD$5b4%64;;;Z;Zr`d  & &4==##p" #,R$(4# VOA|#$58H$H!.7.CilO3B3N_Q/TXN**t}} $ A A ))!"#)*"% ! !-!"#)*"%! *!,M"}R'8&::" &9]1=MBEI$(,1/4&*S ||S !!2!23S E--. S ((9(9: S !)):): ; S "%e.?.?(@"ABS D>S $D>S 'tnS d^S  uU\\"$MM NS rr2c(eZdZdZfdZdZxZS)ElectraDiscriminatorPredictionszEPrediction module for the discriminator, made up of two dense layers.c t|tj|j|j|_t |j|_tj|jd|_ ||_ yNr ) r~rr rrrrr activationr%r^rs r-rz(ElectraDiscriminatorPredictions.__init__ns^ YYv1163E3EF ():):; " &*<*?  6 &&}5==bA rrrrrrrrrs@r-rRrRksOrrRc(eZdZdZfdZdZxZS)ElectraGeneratorPredictionszAPrediction module for the generator, made up of two dense layers.ct|td|_t j |j |j|_t j|j|j |_ y)Ngelurt) r~rrrUr rrrrrrrs r-rz$ElectraGeneratorPredictions.__init__sV (0f&;&;AVAVWYYv1163H3HI rcl|j|}|j|}|j|}|Sr)rrUr)rgenerator_hidden_statesrs r-rz#ElectraGeneratorPredictions.forwards3 #:;  6 }5 rrZrs@r-r\r\~sKJrr\c"eZdZeZeZdZdZdZ y)ElectraPreTrainedModelelectraTct|tjrm|jjj d|j j|j%|jjjyyt|tjrz|jjj d|j j|j2|jj|jjyyt|tjrJ|jjj|jjjdyy)zInitialize the weightsg)meanstdNg?)rJr rr2r[normal_r^initializer_ranger5zero_rrsrfill_)rmodules r- _init_weightsz$ElectraPreTrainedModel._init_weightss fbii ( MM   & &CT[[5R5R & S{{&   &&('  - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  - KK   " " $ MM   $ $S ).rN) rrrr! config_classroload_tf_weightsbase_model_prefixsupports_gradient_checkpointingrlr*rr-rbrbs L0O!&*#*rrbceZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejed<y)ElectraForPreTrainingOutputa Output type of [`ElectraForPreTraining`]. Args: loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss of the ELECTRA objective. logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Prediction scores of the head (scores for each token before SoftMax). hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. NlossrYrrA) rrrrrsrrYr__annotations__rYrrrAr*rr-rrrrsg*)-D(5$$ %,*.FHU&& '.8cd d  "  6 6y. Q#..*K  &',,.s3KTU U!, J%.%:!! @T@TDSC^!3A!6!S^"_ ;; ! !&;&G=R=W=W=Y :  7$68O#P %-).4HQW)X&.2.H.HI_.` +.2 +&&y$++2O2OP %)'#9 (  4- . 33MBM 2"7#B+/!5#%  r) NNNNNNNNNNNNN)rrrrr~rrrrrYrrrrrrrrrrs@r-rvrvsk /0C-11515/3,0048<9==A$(,0/3&*WELL)W!.W!. W u||, W ELL) W  -W ( 5W!) 6W"$u'8'8"9:WD>W$D>W'tnWd^W uU\\"$FF GWWrrvc(eZdZdZfdZdZxZS)ElectraClassificationHeadz-Head for sentence-level classification tasks.czt|tj|j|j|_|j |j n |j}td|_ tj||_ tj|j|j|_ y)Nr^)r~rr rrrclassifier_dropoutrrrUrr num_labelsout_projrr^rrs r-rz"ElectraClassificationHead.__init__?s YYv1163E3EF )/)B)B)NF % %TZTnTn )0zz"45  &"4"4f6G6GH rc |dddddf}|j|}|j|}|j|}|j|}|j|}|S)Nr)rrrUr)rfeatureskwargsrs r-rz!ElectraClassificationHead.forwardIsZ Q1W  LLO JJqM OOA  LLO MM! rrZrs@r-rr<s7IrrceZdZdZdeffd Z ddejdeejdejfdZ xZ S) ElectraSequenceSummarya Compute a single vector summary of a sequence hidden states. Args: config ([`ElectraConfig`]): The config used by the model. Relevant arguments in the config class of the model are (refer to the actual config class of your model for the default values it uses): - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are: - `"last"` -- Take the last token hidden state (like XLNet) - `"first"` -- Take the first token hidden state (like Bert) - `"mean"` -- Take the mean of all tokens hidden states - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2) - `"attn"` -- Not implemented now, use multi-head attention - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction. - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes (otherwise to `config.hidden_size`). - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output, another string or `None` will add no activation. - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation. - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation. r^cft|t|dd|_|jdk(rtt j |_t|drq|jret|dr(|jr|jdkDr |j}n |j}t j|j||_t|dd}|r t|nt j |_t j |_t|dr3|j"dkDr$t j$|j"|_t j |_t|d r5|j(dkDr%t j$|j(|_yyy) N summary_typelastattnsummary_use_projsummary_proj_to_labelsrsummary_activationsummary_first_dropoutsummary_last_dropout)r~rrPrNotImplementedErrorr IdentitysummaryrrrrrrrrU first_dropoutrr last_dropoutr)rr^ num_classesactivation_stringrs r-rzElectraSequenceSummary.__init__nsU #FNFC    && %{{} 6- .63J3Jv78V=Z=Z_e_p_pst_t$// $00 99V%7%7EDL#F,@$GIZN3D$E`b`k`k`m[[] 62 38T8TWX8X!#F,H,H!ID KKM 61 2v7R7RUV7V " 6+F+F GD 8W 2rr cls_indexrc|jdk(r |dddf}n|jdk(r |dddf}n|jdk(r|jd}n|jd k(r|Atj|d ddddf|jd dz tj }nX|j dj d}|jd |jdz z|jdfz}|jd |jd }n|jdk(rt|j}|j|}|j|}|j!|}|S)ak Compute a single vector summary of a sequence hidden states. Args: hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`): The hidden states of the last layer. cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*): Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token. Returns: `torch.FloatTensor`: The summary of the sequence hidden states. rNrwfirstrrer rr.rr|)rwr)rrerY full_likerUr unsqueezerrrgatherrWrrrrUr)rrrrs r-rzElectraSequenceSummary.forwardsn    &"1b5)F   ' )"1a4(F   & ("''A'.F   + - !OO!#rr1*-!''+a/** &//3==bA %,,Uimmo6I-JmN`N`acNdMf-fg "))"i8@@DF   & (% %##F+f%(""6* rr) rrrrr!rrYrrrrrrs@r-rrTsQ2H}H<Y])"..);CEDTDT;U)   )rrz ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ) custom_introc^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S) ElectraForSequenceClassificationct|||j|_||_t ||_t ||_|jyr) r~rrr^rvrcrr8rzrs r-rz)ElectraForSequenceClassification.__init__sH   ++ #F+ 3F; rrrr{rvrrlabelsrr;r<rc | | n|jj} |j|||||||| |  } | d} |j| } d}||jj|j dk(rd|j_nl|j dkDrL|j tjk(s|j tjk(rd|j_nd|j_|jjdk(rIt}|j dk(r&|| j|j}n|| |}n|jjdk(r=t}|| jd|j |jd}n,|jjdk(rt}|| |}| s| f| ddz}||f|zS|St|| | j | j" S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrr{rvrrrr;r<rr regressionsingle_label_classificationmulti_label_classificationrwrsrYrrA)r^rrcr8 problem_typerr}rYrrRr rWr rr rrrA)rrrr{rvrrrrr;r<rXsequence_outputrYrsloss_fctrs r-rz(ElectraForSequenceClassification.forwards(&1%D 'tnD d^D  uU\\"$<< =D D rrz Electra model with a binary classification head on top as used during pretraining for identifying generated tokens. It is recommended to load the discriminator checkpoint into that model. c^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S)ElectraForPreTrainingct||t||_t ||_|j yr)r~rrvrcrRdiscriminator_predictionsrzrs r-rzElectraForPreTraining.__init__s3  #F+ )H)P& rrrr{rvrrrrr;r<rc d| | n|jj} |j|||||||| |  } | d} |j| } d}|t j }|a|j d| jddk(}| j d| jd|}||}|||j}n4|| j d| jd|j}| s| f| ddz}||f|zS|St|| | j| jS)am labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates the token is an original token, - 1 indicates the token was replaced. Examples: ```python >>> from transformers import ElectraForPreTraining, AutoTokenizer >>> import torch >>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator") >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator") >>> sentence = "The quick brown fox jumps over the lazy dog" >>> fake_sentence = "The quick brown fox fake over the lazy dog" >>> fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True) >>> fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") >>> discriminator_outputs = discriminator(fake_inputs) >>> predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) >>> fake_tokens ['[CLS]', 'the', 'quick', 'brown', 'fox', 'fake', 'over', 'the', 'lazy', 'dog', '[SEP]'] >>> predictions.squeeze().tolist() [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] ```Nrrrwr r) r^rrcrr r rrUfloatrrrrA)rrrr{rvrrrrr;r<rXdiscriminator_sequence_outputrYrsr active_loss active_logits active_labelsrs r-rzElectraForPreTraining.forward spZ&1%Q 'tnQ d^Q  uU\\"$?? @Q Q rrz Electra model with a language modeling head on top. Even though both the discriminator and generator may be loaded into this model, the generator is the only model of the two to have been trained for the masked language modeling task. cpeZdZdgZfdZdZdZe ddee jdee jdee jdee jd ee jd ee jd ee jd ee d ee dee de e e jeffdZxZS)rKgenerator_lm_head.weightct||t||_t ||_t j|j|j|_ |jyr) r~rrvrcr\generator_predictionsr rrrgenerator_lm_headrzrs r-rzElectraForMaskedLM.__init__sR  #F+ %@%H"!#6+@+@&BSBS!T rc|jSrrr}s r-get_output_embeddingsz(ElectraForMaskedLM.get_output_embeddings%%%rc||_yrr)rrs r-set_output_embeddingsz(ElectraForMaskedLM.set_output_embeddingss !0rrrr{rvrrrrr;r<rc | | n|jj} |j|||||||| |  } | d} |j| } |j | } d}|Pt j }|| jd|jj|jd}| s| f| ddz}||f|zS|St|| | j| jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Nrrrwr r) r^rrcrrr r rrrrrA)rrrr{rvrrrrr;r<r`generator_sequence_outputprediction_scoresrsrrs r-rzElectraForMaskedLM.forwards(&1%! 667PQ 223DE  **,H-222t{{7M7MNPVP[P[\^P_`D'),CAB,GGF)-)9TGf$ Ev E$1??.99   rr)rrr_tied_weights_keysrrrrrrYrrrrrrrrs@r-rKrKus55&1-11515/3,004)-,0/3&*4 ELL)4 !.4 !. 4 u||, 4 ELL) 4   -4 &4 $D>4 'tn4 d^4  uU\\"N2 34 4 rrKz Electra model with a token classification head on top. Both the discriminator and generator may be loaded into this model. c^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S)ElectraForTokenClassificationc`t|||j|_t||_|j |j n |j }tj||_ tj|j|j|_ |jyr)r~rrrvrcrrr rrrrr8rzrs r-rz&ElectraForTokenClassification.__init__s   ++#F+ )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rrrr{rvrrrrr;r<rc | | n|jj} |j|||||||| |  } | d} |j| } |j | } d}|?  ')HFKKDOO1 'tn1 d^1  uU\\"$99 :1 1 rrceZdZeZdZfdZe ddee jdee jdee jdee jdee jdee jd ee jd ee jd ee d ee d ee de e e jeffdZxZS)ElectraForQuestionAnsweringrcct|||j|_t||_t j |j|j|_|jyr) r~rrrvrcr rr qa_outputsrzrs r-rz$ElectraForQuestionAnswering.__init__sS   ++#F+ ))F$6$68I8IJ rrrr{rvrrstart_positions end_positionsrr;r<rc &| | n|jj} |j||||||| | } | d} |j| }|j dd\}}|j dj }|j dj }d}||t|jdkDr|j d}t|jdkDr|j d}|jd}|jd|}|jd|}t|}|||}|||}||zdz }| s||f| ddz}||f|zS|St|||| j| jS) N)rr{rvrrrr;rr rwr) ignore_indexr9)rs start_logits end_logitsrrA)r^rrcrrMrWrrQrclampr rrrA)rrrr{rvrrrrrr;r<rXrrYrr total_loss ignored_indexr start_lossend_lossrs r-rz#ElectraForQuestionAnswering.forward!s&1%@ 'tn@ d^@  uU\\"$@@ A@ @ rrc^eZdZfdZe ddeejdeejdeejdeejdeejdeejdeejd eed eed eed e e eje ffd Z xZ S)ElectraForMultipleChoicect||t||_t ||_t j|jd|_ |jyrT) r~rrvrcrsequence_summaryr rrr8rzrs r-rz!ElectraForMultipleChoice.__init__gsM  #F+ 6v >))F$6$6: rrrr{rvrrrrr;r<rc L| | n|jj} ||jdn|jd} |!|jd|j dnd}|!|jd|j dnd}|!|jd|j dnd}|!|jd|j dnd}|1|jd|j d|j dnd}|j |||||||| |  } | d} |j | }|j|}|jd| }d}|t}|||}| s|f| ddz}||f|zS|St||| j| jS)a[ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr rwrrrr) r^rrUrrrcrr8r rrrA)rrrr{rvrrrrr;r< num_choicesrXr pooled_outputrYreshaped_logitsrsrrs r-rz ElectraForMultipleChoice.forwardqsX&1%G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  '+ll ))%'/!5#'3 ' #6a8--o> / ++b+6  ')HOV4D%'*Eab*IIF)-)9TGf$ Ev E("5CC2==   rr)rrrrrrrYrrrrrrrrs@r-rres-11515/3,004)-,0/3&*X ELL)X !.X !. X u||, X ELL) X   -X &X $D>X 'tnX d^X  uU\\"$== >X X rrzS ELECTRA Model with a `language modeling` head on top for CLM fine-tuning. c"eZdZdgZfdZdZdZe ddee jdee jdee jdee jd ee jd ee jd ee jd ee jd ee jdee e jdee dee dee dee de ee jeffdZdZxZS)ElectraForCausalLMrc$t|||jstj dt ||_t||_tj|j|j|_ |jy)NzOIf you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.`)r~rrr?warningrvrcr\rr rrrr init_weightsrs r-rzElectraForCausalLM.__init__sj    NNl m#F+ %@%H"!#6+@+@&BSBS!T rc|jSrrr}s r-rz(ElectraForCausalLM.get_output_embeddingsrrc||_yrr)rnew_embeddingss r-rz(ElectraForCausalLM.set_output_embeddingss !/rrrr{rvrrrrrr:rrr;r<rc ||n|jj}| d} |j||||||||| | | | | }|d}|j|j |}d}| *|j || fd|jj i|}|s|f|ddz}||f|zS|St|||j|j|j|jS)a3 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator") >>> config = ElectraConfig.from_pretrained("google/electra-base-generator") >>> config.is_decoder = True >>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NF) rr{rvrrrrr:rrr;r<rrr )rsrYr:rrArB) r^rrcrr loss_functionrrr:rrArB)rrrr{rvrrrrrr:rrr;r<rrrrlm_lossrs r-rzElectraForCausalLM.forwards.R&1%.Bs.nU_j--aZ=N=N1OPns58)rH)rr:rreordered_past layer_pasts ` r-_reorder_cachez!ElectraForCausalLM._reorder_cache>s=) J ncmnn N r)NNNNNNNNNNNNNN)rrrrrrrrrrYrrrrrrrr rrs@r-rrs 55 &0-11515/3,0048<9=)-8<$(,0/3&*S ELL)S !.S !. S u||, S ELL) S   -S  ( 5S !) 6S &S "$u||"45S D>S $D>S 'tnS d^S " uU\\"$EE F#S S lrr) rrKrrrrrrvrbro) discriminator)IrrrA dataclassesrtypingrrrrrrYtorch.utils.checkpointr torch.nnr r r activationsrr generationrmodeling_outputsrrrrrrrrmodeling_utilsr pytorch_utilsrrrutilsrrrconfiguration_electrar! get_loggerrr?roModulerqrrrrrrrr2rRr\rbrrrvrrrrrKrrrr__all__r*rr-rs !99 AA1)   .ll 1   H %Od? ?FC299CN   !" 0ryy0h "))  BII S299SnZ RYYZ zbii&"))$*_**. :+: :8s)ssl 0`RYY`F P '=P  P f[ 2[ [ |H /H H V@ $:@ @ FO "8O O dd 5d d N r/r  rj r