~UhY ddlZddlmZmZmZmZmZddlZddlZddlm Z m Z m Z m Z ddl m cmZddlZddlmZddlmZddlmZmZedd ZGd d ejj4ZGd d ejj4ZGddejj8ZdedfdZGdde j>Z Gdde Z!Gdde Z"Gddejj8Z#dZ$Gdde j4Z%Gdde j4Z&Gd d!e&Z'Gd"d#e&Z(Gd$d%e j>Z)Gd&d'e j>Z*Gd(d)e j>Z+y)*N)AnyOptionalTypeVarUnionoverload)Tensordevicedtypenn) QuantState)GlobalOptimManager)*INVERSE_LINEAR_8BIT_WEIGHTS_FORMAT_MAPPING OutlierTracerTztorch.nn.Module)boundceZdZdZ ddededeedeededed ed eed dffd Z dd Z ddZ ded efdZ xZ S)StableEmbeddinga Custom embedding layer designed to improve stability during training for NLP tasks by using 32-bit optimizer states. It is designed to reduce gradient variations that can result from quantization. This embedding layer is initialized with Xavier uniform initialization followed by layer normalization. Example: ``` # Initialize StableEmbedding layer with vocabulary size 1000, embedding dimension 300 embedding_layer = StableEmbedding(num_embeddings=1000, embedding_dim=300) # Reset embedding parameters embedding_layer.reset_parameters() # Perform a forward pass with input tensor input_tensor = torch.tensor([1, 2, 3]) output_embedding = embedding_layer(input_tensor) ``` Attributes: norm (`torch.nn.LayerNorm`): Layer normalization applied after the embedding. Methods: reset_parameters(): Reset embedding parameters using Xavier uniform initialization. forward(input: Tensor) -> Tensor: Forward pass through the stable embedding layer. Nnum_embeddings embedding_dim padding_idxmax_norm norm_typescale_grad_by_freqsparse_weightreturnc t |||||||||| | tjj || |_t jj|dddiya Args: num_embeddings (`int`): The number of unique embeddings (vocabulary size). embedding_dim (`int`): The dimensionality of the embedding. padding_idx (`Optional[int]`): Pads the output with zeros at the given index. max_norm (`Optional[float]`): Renormalizes embeddings to have a maximum L2 norm. norm_type (`float`, defaults to `2.0`): The p-norm to compute for the `max_norm` option. scale_grad_by_freq (`bool`, defaults to `False`): Scale gradient by frequency during backpropagation. sparse (`bool`, defaults to `False`): Computes dense gradients. Set to `True` to compute sparse gradients instead. _weight (`Optional[Tensor]`): Pretrained embeddings. r weight optim_bits N) super__init__torchr LayerNormnormr get_instanceregister_module_override) selfrrrrrrrrr r __class__s i/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/bitsandbytes/nn/modules.pyr$zStableEmbedding.__init__2sp@            HH&&}V&D '')BB4T`bdSefctjjj|j|j yNr%r initxavier_uniform_r _fill_padding_idx_with_zeror*s r,reset_parametersz StableEmbedding.reset_parametersa(  %%dkk2 ((*r-c|jFtj5|j|jj ddddyy#1swYyxYwNrrr%no_gradr fill_r4s r,r3z+StableEmbedding._fill_padding_idx_with_zerolS    ' 7 D,,-33A6 7 7 ( 7 7 )AAinputc ^tj||j|j|j|j |j |j}|jtj}|j|j|jjSr/) F embeddingr rrrrrtor%get_default_dtyper'r r*r>embs r,forwardzStableEmbedding.forwardqs}kk  KK    MM NN  # # KK ffU,,./yy~  !2!233r-)NN@FFNNNrN)__name__ __module__ __qualname____doc__intrfloatboolrr$r5r3rF __classcell__r+s@r,rrs:&*$(#($(-g-g-gc] -g 5/ -g  -g!-g-g&!-g -g^+7 4V44r-rceZdZdZ ddededeedeededed ed eed ee d dffd Z ddZ ddZ ded efdZ xZS) EmbeddingzS Embedding class to store and retrieve word embeddings from their indices. Nrrrrrrrrr rc t ||||||||||  tjj |dddiyr)r#r$r r(r)) r*rrrrrrrrr r+s r,r$zEmbedding.__init__sV>            '')BB4T`bdSefr-ctjjj|j|j yr/r0r4s r,r5zEmbedding.reset_parametersr6r-c|jFtj5|j|jj ddddyy#1swYyxYwr8r9r4s r,r3z%Embedding._fill_padding_idx_with_zeror<r=r>c tj||j|j|j|j |j |j}|Sr/)r@rAr rrrrrrDs r,rFzEmbedding.forwardsHkk  KK    MM NN  # # KK  r-)NNrGFFNNrH)rIrJrKrLrMrrNrOrr r$r5r3rFrPrQs@r,rSrSs&*$(#($(#'*g*g*gc] *g 5/ *g  *g!*g*g&!*g *g *gX+7 V  r-rSceZdZddddddejddf deej deeded e d e d ejd ed de ddfdZ dZ dZdZdZe d&dej dee efde d ed ddf dZed'dZdZdZd(deeeee fde fdZd(deeeee fde fdZe d)dedeeeefd eee e fde def d!Zed*ded ee e fde defd"Zed*ded#ede defd$Zfd%ZxZS)+ Params4bitNF@Tfp4data quant_state blocksizecompress_statistics quant_type quant_storagemodule Linear4bit bnb_quantizedrc |tjd}tjj|||} || _|| _|| _|| _|| _| | _ || _ || _ | Sr8) r%emptyr_make_subclassr^r_r`r]rardr\rb) clsr\ requires_gradr]r^r_r`rarbrdr*s r,__new__zParams4bit.__new__so <;;q>D||**3mD"#6 $&**   r-cv|jj}|j|d<|j|d<|S)Nr\ri)__dict__copyr\rir*states r, __getstate__zParams4bit.__getstate__s6 ""$ f !%!3!3o r-c|d|_|d|_|d|_|d|_|d|_|d|_|d|_|d|_|d |_y) Nrir^r_r`r]r\rardrb) rir^r_r`r]r\rardrbrns r, __setstate__zParams4bit.__setstate__sr"?3{+#()>#?  - /&M "?3"?3Ho r-ct|jt|}|j}|j|t j |d|_t j |d|_|S)Nr]r\)typerjrprrrmdeepcopyr]r\)r*memo new_instanceros r, __deepcopy__zParams4bit.__deepcopy__sgDz))$t*5 !!#!!%(#'==}1E#F   MM%-8 r-ct|jt|}|j}|j||Sr/)rtrjrprr)r*rwros r,__copy__zParams4bit.__copy__s<Dz))$t*5 !!#!!%(r-quantized_statsric tjj||j|}||_t j |||_|jj|_|jj|_ |jj|_ d|_ |j|_||_|j|j|j_|S)N)qs_dictr T)r%rrgrBrir from_dictr]r^nestedr_r`rdr rarb)rhr\r{rir rbkwargsr*s r,from_prequantizedzParams4bit.from_prequantized s||**3@*%//PVW))33#'#3#3#:#: **55!!ZZ ;; "&*&6&6DKK # r-c|i}tjj5||i|cdddS#1swYyxYwr/)r%_CDisableTorchFunctionSubclass)rhfunctypesargsrs r,__torch_function__zParams4bit.__torch_function__%s> >F XX 2 2 4 )(( ) ) )s5>cZ|jjj|}tjj ||j |j|j|j\}}||_||_ |j||j_ d|_ |S)N)r^r_r`raT) r\ contiguousrBbnb functional quantize_4bitr^r_r`rar]rbrd)r*r ww_4bitr]s r, _quantizezParams4bit._quantize,s II " % %f -!nn:: nn $ 8 8,, ;   & ;; "&1DKK #! r-c&|jdSNcpurrBr4s r,rzParams4bit.cpu<wwew$$r-r non_blockingc<|j|d|S||SNcudar rrr*r rs r,rzParams4bit.cuda?'wwfQ]w^^FQ]w^^r-c<|j|d|S||SNxpurrrs r,rzParams4bit.xpuB'wwv~eP\w]]6P\w]]r-r*r cyr/r*r r rs r,rBz Params4bit.toE r-cyr/rr*r rs r,rBz Params4bit.toMNQr-tensorcyr/rr*rrs r,rBz Params4bit.toPDGr-c tjjj|i|\}}}}|,|jdk7r|j s|j |S|j|jj|tt|!||||j|j|j|j|j|j}|S)Nmetar r r)rir]r^r_r`ra)r%r_nn _parse_tortrdrr]rBrYr#rir^r_r`ra r*rrr r rconvert_to_format new_paramr+s r,rBz Params4bit.toSs9>9O9OQU9`Y_9`6|%6  &++"7@R@R>>&) )+  ##F+" &L Q"00 ,,..$($<$<??"00I r-)FrN)rNNF....) rIrJrKr%uint8rrr rMrOstrr rjrprrrxrz classmethoddictrrrrrrr rrrrrBrPrQs@r,rYrYs<(,,0$(%*[[)-#u||$j)    " {{& 4 &  $)- llc3h  &  2))  %_8E#vs*:$;<_SW_^(5fc)9#:;^RV^03-0 sF{+,eSj)*  QQ5,QDQ1QQ GGFG$GGGr-rYrb) Embedding4bitrcclt|jddyt|ddtjd|jjddk(sJt |jt s't |j|jd|_|j|j_y)Nr]zhFP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first.T)rard) getattrr warningswarnshape isinstancerYrar])rbs r,'fix_4bit_weight_quant_state_from_modulerisv}}mT2>v}d+3 v ==  q !Q && & fmmZ 0"6==@T@Tdhi & 2 2FMMr-cteZdZdZddddej dffd ZdZfdZdejfd Z xZ S) rca This class is the base module for the 4-bit quantization algorithm presented in [QLoRA](https://arxiv.org/abs/2305.14314). QLoRA 4-bit linear layers uses blockwise k-bit quantization under the hood, with the possibility of selecting various compute datatypes such as FP4 and NF4. In order to quantize a linear layer one should first load the original fp16 / bf16 weights into the Linear4bit module, then call `quantized_module.to("cuda")` to quantize the fp16 / bf16 weights. Example: ```python import torch import torch.nn as nn import bitsandbytes as bnb from bnb.nn import Linear4bit fp16_model = nn.Sequential( nn.Linear(64, 64), nn.Linear(64, 64) ) quantized_model = nn.Sequential( Linear4bit(64, 64), Linear4bit(64, 64) ) quantized_model.load_state_dict(fp16_model.state_dict()) quantized_model = quantized_model.to(0) # Quantization happens here ``` TNr[c t |||||t|jjd|||||_||_d|_d|_||_y)aw Initialize Linear4bit class. Args: input_features (`str`): Number of input features of the linear layer. output_features (`str`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. Frir_r`rarbN) r#r$rYr r\ compute_dtypecompute_type_is_setr]ra) r*input_featuresoutput_featuresbiasrr_r`rar r+s r,r$zLinear4bit.__init__sb, $G KK   3!'   +#( *r-cd|jtjtjfvr|j|_y|jtj k(r|jtjk(rL|j |jdk(r,tjdtjdd|jtjk(rN|j |jdk7r-tjdtjddyyyy)NzInput type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference.ignorez .*inference.)messagezInput type into Linear4bit is torch.float16, but bnb_4bit_compute_dtype=torch.float32 (default). This will lead to slow inference or training speed.z.*inference or training) r r%float32bfloat16rfloat16numelrrrfilterwarnings)r*xs r,set_compute_typezLinear4bit.set_compute_types 77u}}enn5 5"#D  WW %!!U]]2 QWWR[8P Y''.I!!U]]2 QWWR[8P k'':ST 9Q2&r-c t||||t|jddZ|jjj dj D]"\}}|r|n|j||dz|z<$yy)zc save weight and bias, then fill state_dict with components of quant_state r]NT)packedzweight.)r#_save_to_state_dictrr r]as_dictitemsdetach)r* destinationprefix keep_varskvr+s r,rzLinear4bit._save_to_state_dicts #KC 4;; t 4 @ //77t7DJJL U1;Da!((* FY.23 U Ar-rct||ja|jj|jk7r>|jjj |j|j_|j s|j |d|_|j}|j|j |j}|jdn$|jj |j}tj||jj||jjj |S)NT)rr]) rrr r\rBrrrr matmul_4bitr tr])r*r inp_dtypers r,rFzLinear4bit.forwards/5 99 TYY__%?!YY^^..qww7DIIN''  ! !! $'+D $GG    )T''(Ayy(tdiill4;M;M.Nq$++--/$++JaJabeefoppr-) rIrJrKrLr%rr$rrrrFrPrQs@r,rcrczsEH kk#+JU( Uqqr-rccBeZdZdZdddej dffd ZxZS) LinearFP4z' Implements the FP4 data type. TNc 2t||||||d||y)Q Args: input_features (`str`): Number of input features of the linear layer. output_features (`str`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. r[Nr#r$ r*rrrrr_rar r+s r,r$zLinearFP4.__init__*&          r-rIrJrKrLr%rr$rPrQs@r,rrs' kk  r-rcBeZdZdZdddej dffd ZxZS) LinearNF4a"Implements the NF4 data type. Constructs a quantization data type where each bin has equal area under a standard normal distribution N(0, 1) that is normalized into the range [-1, 1]. For more information read the paper: QLoRA: Efficient Finetuning of Quantized LLMs (https://arxiv.org/abs/2305.14314) Implementation of the NF4 data type in bitsandbytes can be found in the `create_normal_map` function in the `functional.py` file: https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L236. TNc 2t||||||d||y)rnf4Nrrs r,r$zLinearNF4.__init__#rr-rrQs@r,rrs'  kk  r-rc eZdZ ddeej deej deej fdZfdZdZddee e e e fde fd Zddee e e e fde fd Zd Ze dd edee e e fd ee ee fde def dZedd ed e ee fde defdZedd edede defdZfdZxZS) Int8Paramsr\CBSCBc|tjd}tjj|||}||_||_||_|Sr8)r%rfrrgrrhas_fp16_weights)rhr\rirrrobjs r,rjzInt8Params.__new__CsI <;;q>Dll))#t]C/ r-c |jrt| |S|jj j|t j }tjj|\}}}||_||_ ||_ |S)Nr r ) rr#rBr\rr%rrrint8_vectorwise_quantrr)r*r Brr_r+s r,rzInt8Params._quantizeSsw  7:f% % II " % %V5== % I^^99!< C  r-c&|jdSrrr4s r,rzInt8Params.cpu`rr-r rc<|j|d|S||Srrrs r,rzInt8Params.cudacrr-c<|j|d|S||Srrrs r,rzInt8Params.xpufrr-c 4t|jt|tj|j||j |j tj|j|tj|j|}|S)N)r\rirrr) rtrjrmrur\rirrr)r*rvrws r,rxzInt8Params.__deepcopy__isrDz)) Jtyy$/,,!22}}TWWd+ dhh- * r-r*r rcyr/rrs r,rBz Int8Params.tourr-cyr/rrs r,rBz Int8Params.to}rr-rcyr/rrs r,rBz Int8Params.torr-ctjjj|i|\}}}}|C|jdk7r4|j j jdk(r|j|Stt|)||||j|j}|j|_ |j|_|S)Nrrr)rir)r%rrrrtr\r rrr#rBrirrrrs r,rBz Int8Params.tos9>9O9OQU9`Y_9`6|%6  &++"7DII>&) )" &L Q"00!%!6!6I  77IL HHIM r-)NTFNNrrr)rIrJrKrr%rrjrrrrMr rrOrrrxrrr rBrPrQs@r,rrBsq(,%)&* u||$ U\\ "  ell #  %_8E#vs*:$;<_SW_^(5fc)9#:;^RV^ 03-0 sF{+,eSj)*  QQ5,QDQ1QQ GGFG$GGGr-rcd|j|d}|y|j|dd}t|tjr|j }t|t r|tvrtd|t|t r|tvr t|}|dk7rtd|y)Nr weight_formatrowz'Expected supported weight format - got z+Only 'row' weight format is supported, got ) getpoprr%ritemrMr ValueError) state_dictrlocal_metadatastrict missing_keysunexpected_keys error_msgsr rs r,maybe_rearrange_weightrs ^^vhf- .F ~NNfX]#;UCM-.%**, -%-?i*iB=/RSS M3 'M=g,gB=Q F}oVWWr-c:eZdZdZdfd ZdZdedefdZxZS) Embedding8bita This class implements [LLM.int8()](https://arxiv.org/abs/2208.07339) algorithm for embedding layer Quantization API is similar to Linear8bitLt: ```python import torch import torch.nn as nn from bitsandbytes.nn import Embedding8bit fp16_module = nn.Embedding(128, 64) int8_module = Embedding8bit(128, 64) int8_module.load_state_dict(fp16_module.state_dict()) int8_module = int8_module.to(0) # Quantization happens here ``` ct||||||jjj|_t |jjdd|_y)NrFrri)r#r$r r\r r)r*rrr r r+s r,r$zEmbedding8bit.__init__sK vUS[[%%++  !1!1EY^_ r-ctd)Nz.Saving Embedding8bit module is not implementedNotImplementedErrorr*rrrs r,rz!Embedding8bit._save_to_state_dict!"RSSr-r>rct|jds td|jj}|jj}|j |j |jfk(sJ|j |j fk(sJtj||}tj||j|j d}||dz z}|j|jS)NrzKEmbedding layer is not quantized. Please call .cuda() or .to(device) first.rg_@) hasattrr RuntimeErrorr\rrrrr@rAviewrBr )r*r>rows row_statscompressed_outputcompressed_output_statsoutputs r,rFzEmbedding8bit.forwardst{{E*lm m{{KKOO zzd1143E3EFFFF4#6#6"8888KKt4"#++eY^^DDWDWYZ5["\"&=&EFyy$$r-)NN) rIrJrKrLr$rrrFrPrQs@r,rrs'&` T%V%%r-rcbeZdZdZddej dffd ZdefdZdZ dedefd Z xZ S) ra3 This is the base class similar to Linear4bit. It implements the 4-bit quantization algorithm presented in [QLoRA](https://arxiv.org/abs/2305.14314) for embeddings. Quantization API is similar to Linear4bit: ```python import torch import torch.nn as nn from bitsandbytes.nn import Embedding4bit fp16_module = nn.Embedding(128, 64) quantized_module = Embedding4bit(128, 64) quantized_module.load_state_dict(fp16_module.state_dict()) quantized_module = quantized_module.to(0) # Quantization happens here ``` Nr[c@t||||||jjj|_t |jjdd||||_|jj }||zdk7rtjd|d|dyy)NrFrrzEmbedding size z is not divisible by block size z#. This will lead to slow inference.) r#r$r r\r rYr^rr) r*rrr r`rar r^r+s r,r$zEmbedding4bit.__init__s vUS[[%%++  KK   $!'   KK)) 9 $ ) MM!-0PQZP[\44  *r-r>c|j|jjjzdk(sJ|jjj t jj |j|jzdzd}t jjj|j |j|jdz|j dd}|j|j|jzdzdfk(sJ|j|jjz}|jjj}|j|j|zfk(sJt jjj|j |j||j d}|j|j|zfk(sJtj |jj}||_t j"g|j|j|_ t$jj'||}|jg|j|jk(sJ|j)|j*S)Nrrr r>r)rr r]r^r\r r%rrr rrArrabsmaxrmruSizerdequantize_4bitrBr ) r*r> w_4bit_uint8 output_4bitblocks_per_embr+ output_absmaxoutput_quant_stater%s r, _forward_with_partial_dequantizez.Embedding4bit._forward_with_partial_dequantize sI!!DKK$;$;$E$EEJJJ{{'',,U[[9>>t?R?RUYUgUg?gkl?lnop hh))33$$T%8%8$:L:LPQ:QR4  $r1+   U[[]T5G5G%G1%La$PPPP++t{{/D/DD((//|| 3 3n DFFFF++55;;t22NC6  $    ""u{{}~'E&GGGG!]]4;;+B+BC$1!#(::.P .PT=O=O.P#Q // =OP||AAd.@.@AAAAyy$$r-ctd)Nz.Saving Embedding4bit module is not implementedrrs r,rz!Embedding4bit._save_to_state_dict+rr-rct||j|jjjzdk(r|j |St jj|jj|jj}tjjj||j|jS)Nrr*)rrr r]r^r3rrr-r\r%r rArBr )r*r>dequantized_weights r,rFzEmbedding4bit.forward.s/5    7 7 A A AQ F88? ? ^^;;DKKr-r@creZdZdZ d dedeffd ZfdZfdZdZde jfd Z xZ S) Linear8bitLtaZ This class is the base module for the [LLM.int8()](https://arxiv.org/abs/2208.07339) algorithm. To read more about it, have a look at the paper. In order to quantize a linear layer one should first load the original fp16 / bf16 weights into the Linear8bitLt module, then call `int8_module.to("cuda")` to quantize the fp16 weights. Example: ```python import torch import torch.nn as nn import bitsandbytes as bnb from bnb.nn import Linear8bitLt fp16_model = nn.Sequential( nn.Linear(64, 64), nn.Linear(64, 64) ) int8_model = nn.Sequential( Linear8bitLt(64, 64, has_fp16_weights=False), Linear8bitLt(64, 64, has_fp16_weights=False) ) int8_model.load_state_dict(fp16_model.state_dict()) int8_model = int8_model.to(0) # Quantization happens here ``` rrcVt|||||tj|_||_||j_||j_|dkDr|sd|j_t|jj|||_ |jty)ay Initialize Linear8bitLt class. Args: input_features (`int`): Number of input features of the linear layer. output_features (`int`): Number of output features of the linear layer. bias (`bool`, defaults to `True`): Whether the linear class uses the bias term as well. TrN)r#r$r MatmulLtStateroindex thresholdruse_poolrr r\"_register_load_state_dict_pre_hookr) r*rrrrrHrGr r+s r,r$zLinear8bitLt.__init__s* $G&&(  ( &6 # s?#3"&DJJ  !1!1DTdtu  //0FGr-ct ||||d}t|j|}t|j|}||z}|dz}|jj s|@|r|n|j ||<tjdtj||<y|@|r|n|j ||<tjdtj||<yyy)Nrrr)r ) r#rrr rorrr%rr) r*rrrscb_nameparam_from_weightparam_from_statekey_name format_namer+s r,rz Linear8bitLt._save_to_state_dicts #KC$DKK:"4::x8xj). zz** ,=F(9L]LdLdLf H%+0<<+M K(!->-%)[[__DJJN&&s+! ,r-c|jj|j_|jj|j_d|j_d|j_yr/r rrorr4s r,init_8bit_statezLinear8bitLt.init_8bit_state>     r-rcr|j|j_|jj|j |j a|j j|jk7r>|j jj|j|j _tj||j|j |j}|jjs;|jj%|jj|j_|SN)rro) trainingro is_trainingr rr]rr r\rBrmatmulrr*routs r,rFzLinear8bitLt.forwards!%  ;;>> %  " 99 TYY__%?!YY^^..qww7DIINjjDKKdiitzzJzz**tzz}}/H#zz}}DKK  r-)TTrENN) rIrJrKrLrMr$rrRr]r%rrFrPrQs@r,rCrCbsVF H H HDN0%,N r-rCc2eZdZdfd ZdZdZdZxZS)OutlierAwareLinearcFt|||||d|_d|_yr)r#r$ outlier_dim is_quantized)r*rrrr r+s r,r$zOutlierAwareLinear.__init__s& $G!r-ctd)NzJPlease override the `forward_with_outliers(self, x, outlier_idx)` functionr)r*r outlier_idxs r,forward_with_outliersz(OutlierAwareLinear.forward_with_outlierss!"noor-ctd)NzEPlease override the `quantize_weights(self, w, outlier_idx)` functionr)r*rrls r,quantize_weightz"OutlierAwareLinear.quantize_weights!"ijjr-c||jQtj}|js t d|j |j }||_|jsS|j|j |j}|j jj|d|_yy)NzTPlease use OutlierTracer.initialize(model) before using the OutlierAwareLinear layerT) rirr(is_initializedprint get_outliersr rjror\rU)r*rtracerrlrs r,rFzOutlierAwareLinear.forwards    #"//1F((*lm --dkk:K*D   $$T[[$2B2BCA KK   " "1 % $D !r-)TN)rIrJrKr$rmrorFrPrQs@r,rgrgs" pk %r-rgc8eZdZ dfd ZdZdZxZS)SwitchBackLinearBnbc Nt |||||tj|_||_||j_||j_||j_|dkDr|sd|j_ t|jj|||_ y)NrETr) r#r$rrFrorGrHrmemory_efficient_backwardrIrr r\) r*rrrrrxrHrGr r+s r,r$zSwitchBackLinearBnb.__init__s $G&&(  ( &6 #/H , s?#3"&DJJ  !1!1DTdtu r-c|jj|j_|jj|j_d|j_d|j_yr/r\r4s r,r]z#SwitchBackLinearBnb.init_8bit_state-r^r-c2|j|j_|jj|j t j|j|jjd|j|jz}yr`) rarorbr rr]r matmul_mixedhalfrrds r,rFzSwitchBackLinearBnb.forward3se!%  ;;>> %  "qvvx)9)9);$djjY\`\e\eer-)TTFrENN)rIrJrKr$r]rFrPrQs@r,rvrvs( "'v. fr-rv),rmtypingrrrrrrr%rr r r torch.nn.functionalrr@ bitsandbytesrbitsandbytes.functionalr bitsandbytes.optimr bitsandbytes.utilsrrrrSr ParameterrYrLinearrcrrrrrrr8r@rCrgrvrr-r,rs[  :: ++.1  C()g4ehh((g4TL""L^U##Up3E:W4X3"vqvqr! ! H( ( VO##OdX*,%BLL,%^aBLLaH = & = &U299Up%%6$f"))$fr-