|Uh#I,tddlmZddlZddlZddlmZddlmZm Z m Z ddl Z ddl Z ddl mZddlmZddlmZmZdd lmZiZ ej.ej0ej2fej4ej6fej8ej:fej<ej>ej@fejBejDfej.ej0ej2fejFejHejJfd Z&ejNejPfejRejTfejVejXfejZej\fejNejPfejRejTfd Z/ej`ejbejdfejfejhejjfejlejnejpfejrejtejvfejxejzej|fej~ejejfd ZBGd dZCGddZDGddZEe jddZGe jjdkDrde jfdZJnddlKZKde jfdZJe jeGddZMdde jfdZNddZOddZPddZQdd ZRdd!ZSdd"ZTdd#ZUed$eV%dd&ZWd'ee e jfd(ZXd)ed*ejfd+ZZde ed*e ejfd,Z[ed$eV% dded-e e jd.e\d*efd/Z]Gd0d1Z^ dde jd2e e jd3e e jd-e e jd*e_e je^ff d4Z` dde jd5e e^d3e e jd2e e jd-e e jd6ead*e jfd7Zbdd9Zcddd8de jfde jd3e e jd-e e jfd:Zeddd8de jfde jd3e e jd-e e jfd;Zfddd8ddZh dde jd5e e^d3e e jd-e e jd6ead*e jf d?Zi dde jd5e e^d3e e jd-e e jd6ead*e jf d@Zjed$eV% dded2e e jd-e e jd*e_ee_eefffdAZked$eV% ddedBe e_eefd3e e jd2e e jd-e e jd*ef dCZled$eV%dded2ed-e e jd*efdDZmed$eV%dded2ed-e e jd*efdEZn ddFeodGedHedIedJe\dKe\dLeadMe\dNe e jdOe\dPe\dQe\dRe\dSe\dTe e jdUe\d*df"dVZpedWeV% ddFeodGedHedIedNe e jdJe\dOe\dKe\dLeadMe\dXedYe e jdZed[e e jd\ed]e e jdRe\dSe\dTe e jdUe\d*df*d^Zq ddFeodGedHedIedNe e jdJe\dOe\dPe\dQe\dKe\dLeadMe\dXedYe e jd_ed`e e jdRe\dSe\d*df&daZred$eV%ddbedcedLeaddeafdeZsed$eV%dfedgedhediefdjZte jfdkZv ddedled-e e jfdmZw ddedled-e e jfdnZx ddedled-e e jfdoZyde jfde jdle jd-e e jfdpZ{ dde jdqe jdre jd-e e jdse e jf dtZ|ed$eV% dde jdqe e jdre e jdue e jd*e_e je je e jff dvZ}ed$eV%dde jfdwZ~GdxdyZGdzd{ZGd|d}Zd~ZdZe jfdZ dde jdre e jdqe e jde e jde e jf dZde jde jfdZdde jfdZ dde ee jfdle jd-e e jfdZddZdZedeV%ddZed$eV%e jdfdZy))IterableN)prod)AnyOptionalUnion)Tensor) deprecated)pack_dict_to_tensorunpack_tensor_to_dict)lib)adammomentumrmsproplionadagradlambademamix)rrrrrlars)rrrrrrc4eZdZdZdZdZedZddZy)GlobalPageManagerNctdNzCall get_instance() instead RuntimeErrorselfs i/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/bitsandbytes/functional.py__init__zGlobalPageManager.__init__z899cg|_yN) paged_tensorsrs r initializezGlobalPageManager.initialize}s r!c|j0|j||_|jj|jSr# _instance__new__r%clss r get_instancezGlobalPageManager.get_instance6 == KK,CM MM $ $ &}}r!cJ|jdddD]}t||y)N)r$prefetch_tensor)rto_cputs r prefetch_allzGlobalPageManager.prefetch_alls,##DbD) 'A Av & 'r!F) __name__ __module__ __qualname__r(rr% classmethodr,r3r!rrrws*I:  'r!rc2eZdZdZdZdZedZdZy)CUBLAS_ContextNctdrrrs rrzCUBLAS_Context.__init__r r!ci|_yr#)contextrs rr%zCUBLAS_Context.initializes  r!c|j0|j||_|jj|jSr#r'r*s rr,zCUBLAS_Context.get_instancer-r!c|j|jvrtjj }tjj |t jtj|j|j<tjj ||j|jSr#) indexr>torchcudacurrent_device set_devicectc_void_pr get_context)rdevice prev_devices rrHzCUBLAS_Context.get_contexts} <rs rr%zCusparse_Context.initializes{{3#3#3#56 r!c|j0|j||_|jj|jSr#r'r*s rr,zCusparse_Context.get_instancer-r!)r5r6r7r(rr%r8r,r9r!rrLrLs%I:7r!rLrC)rAac@tjj|Sr#)rBrC device_ofrQs r_cuda_device_ofrUszz##A&&r!c*tjSr#) contextlib nullcontextrTs rrUrUs%%''r!dtyperIc|jt|z}tjt j |}t j |t jtj}tjj||}tj||t|j|}d|_|j |_|S)N)shape)rZcountT)itemsizerr cget_managed_ptrrFc_size_tcastPOINTERc_intnp ctypeslibas_arrayrB frombufferviewis_pagedrA page_deviceid)rZrIr\ num_bytescuda_ptrc_ptr new_arrayouts r get_pagedrpse,I##BKK $:;H GGHbjj2 3E %%e5%9I   9Ee E J J5 QCCL C Jr!FAc|jsJd|rd}n |j}tjt |t j |jt j|y)Nz%Only paged tensors can be prefetched!r/) rirjr cprefetchget_ptrrFr`nbytesc_int32)rqr1deviceids rr0r0sO ::>>>: ??MM'!*bkk!((3RZZ5IJr!c d}|jtjk(r+ttd|dd}t j |}nG|jtjk(r*ttd|dd}t j|}|td|t|dd}|r|rt|| t||t|t|t j|j|js |jrtjj!yy)Nc_fp32_uint8zFunction not implemented: is_managedF)rZrBfloat32getattrr rFc_floatuint8c_uint8NotImplementedErrorr0rtc_int64numelrirC synchronize) func_namerqBvalueprefetchfunccvaluer|s relementwise_funcrs Dww%--sa {%0$7E" EKK sa {&148E" |!$>yk"JKKL%0Jh = A WQZAGGI)>?zzQZZ    r!c td|d|y)Nfillr)rqrrIrs rrrsVQe,r!c td||dy)N_mulrr)rqrrIs rrrsVQ1%r!cN|rdnd}d|z}|s|dkr|sd|znd|zdz }tj|d|}d|jz }|dk(r|S|jdz}tj|d|j dg|zz||dj zS) Nr ?r)rBlinspacerrtolist)signed total_bitsadd_zerosign total_valuesvaluesgapls rcreate_linear_maprs4sDj=L:> -3q*}: 8I ^^D#| 4F   C ax LLNa ||F2AJ--/1#);fQRj>O>O>QQRRr!c ddlm}|rv|jt j |ddddj }dgdz}|jt j |dddd j }nu|jt j |ddddj }dgd z}|jt j |dddd j }||z|z}t j|}|jj}||jz}|jd k(sJ|S#t$r}td|d}~wwxYw) Nr)normzZScipy is required for `create_normal_map`. Install `bitsandbytes` with the `[test]` extra.g? r/rr) scipy.statsr ImportErrorppfrBrrrsortrmaxr) offsetuse_extra_valueriev1v2v3vrs rcreate_normal_maprsR$  XXennVS!4Sb9 : A A CSH xxvsA6s;<< D D F XXennVS!4Sb9 : A A CSH xxvsA6s;<< D D F R" A \\!_F [[] ! !F fjjlF <<>S  M/  h  sE E0 E++E0cT|}|}|rdnd}||z||z k(sJg}g}ttd||z z d||z zdD]\} } |jd| zg} tt j ddg|} d|dz z} td|zD]}| D]z}|dk7rdnd}tt|D]\} }||d| dz zzz }|dk(r |d| zz}n|d|| z dz zz}| j||si| j| |t | d|zk(sJ| j|dkr/dt | z }t|D]} | jd| jtj| }||jz}|S)Nr rr)repeatrr) enumeraterangeappendlist itertoolsproductlenrrBtensorr)r exponent_bitsprecision_bitsrephas_signevaluespvaluesivalrlstbiasevalue bit_patternrpvalrcodes rcreate_fp8_mapr2sAAqH q5J) )) )GGEA-(*B$C"DaM\dLdFeghij3q#vF y  !Q? @C " #Dm,- & &K1A!E$T+%67 04Ah// 0{T7 *v}q'8%9 99 MM%  uf% & & v;!Z- '' ' KKMA~CKs A MM!   KKM << DDHHJD Kr!cg}|dz }d||z zdz }t|D]}t|rd||z|z zdznd||z|z dzzdz}tjdd|tj}|dd|ddzdz } |d|dz |zz| zj z }|s|d|dz |zz | zj z }|d kDr{tjdd|dztj}|dd|ddzdz } |d|dz zz| zj z }|r!|d|dz |zz | zj z }|j d |j d t|d|zk(sJd t|z } t| D]}|j d |jtj|tjS) a+ Creates the dynamic quantiztion map. The dynamic data type is made up of a dynamic exponent and fraction. As the exponent increase from 0 to -7 the number of bits available for the fraction shrinks. This is a generalization of the dynamic type where a certain number of the bits and be reserved for the linear quantization region (the fraction). n determines the maximum number of exponent bits. For more details see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561] r rg?rZNr/@ rrr) rintrBrr}rrrrr) rmax_exponent_bitsrdata non_sign_bitsadditional_itemsrfraction_items boundariesmeansrs rcreate_dynamic_mapr]s-" DNM]->>?!C $ % O !m#&77 81 <q=(+<1S8 ",q01A56%?GGII  r 1A 56:;1S8 ",q01A56%?GGII  r 1A 56:;A  AHHQK A,C 3Z  FFHq A AEEGKKMA Hr!tensorscd}t}|D]G}|t|ddr||jz}|j|jj I|s2t d|Dcgc]}|j|jfc}t|dkDr2t d|Dcgc]}|j|jfc}|Scc}wcc}w)apVerifies that the input tensors are all on the same device. An input tensor may also be marked as `paged`, in which case the device placement is ignored. Args: tensors (`Iterable[Optional[torch.Tensor]]`): A list of tensors to verify. Raises: `RuntimeError`: Raised when the verification fails. Returns: `Literal[True]` TriFzZAll input tensors need to be on the same GPU, but found some tensors to not be on a GPU: r zcInput tensors need to be on the same GPU, but found the following tensor and device combinations: ) setr~is_cudaaddrIrArr\r)ron_gpugpu_idsr2s r is_on_gpurs#FeG ( =J!> aii F KK ' ( iIPkQDElmlslsuvu}u}k~kQjR S   7|arRYtZMNuvu|u|~GGuHtZs[ \   MkQ tZs %C %C rreturnctjtjj |j j Sr#)rFrGrB_C_cuda_getCurrentRawStreamrIrA)rs r_get_tensor_streamrs* ;;uxx99&--:M:MN OOr!cN|ytj|jS)zGets the memory address of the first element of a tenso Args: A (`Optional[Tensor]`): A PyTorch tensor. Returns: `Optional[ct.c_void_p]`: A pointer to the underlying tensor data. N)rFrGdata_ptr)rqs rrtrts! y ;;qzz| $$r!rorc :|jdkrtd|jd|dkDrtd||dkr |dk(rdd|zz }|0tjd tj|j }t |5t||g|jtjk(r_tjt|t|tj|tj|jn|jtjk(r_tj t|t|tj|tj|jntd |jddd|dkrQt#d|z }tj$d d |j'j)|j }||}|S#1swYaxYw)a Estimates 256 equidistant quantiles on the input tensor eCDF. Uses SRAM-Quantiles algorithm to quickly estimate 256 equidistant quantiles via the eCDF of the input tensor `A`. This is a fast but approximate algorithm and the extreme quantiles close to 0 and 1 have high variance / large estimation errors. These large errors can be avoided by using the offset variable which trims the distribution. The default offset value of 1/512 ensures minimum entropy encoding -- it trims 1/512 = 0.2% from each side of the distrivution. An offset value of 0.01 to 0.02 usually has a much lower error but is not a minimum entropy encoding. Given an offset of 0.02 equidistance points in the range [0.02, 0.98] are used for the quantiles. Parameters ---------- A : torch.Tensor The input tensor. Any shape. out : torch.Tensor Tensor with the 256 estimated quantiles. offset : float The offset for the first and last quantile from 0 and 1. Default: 1/(2*num_quantiles) num_quantiles : int The number of equally spaced quantiles. Returns ------- torch.Tensor: The 256 quantiles in float32 datatype. rzQQuantile estimation needs at least 256 values in the Tensor, but Tensor had only z values.zgCurrently only a maximum of 256 equally spaced quantiles are supported, but the argument num_quantiles=`?r rN)rrYzNot supported data type r)rrrBzerosr}rIrUrrZr cestimate_quantiles_fp32rtrFrrcfloat16cestimate_quantiles_fp16roundrlongto)rqrorrstepidxs rrrsF wwy3!_`a`g`g`i_jjr s  s!uwDvE F  sv2a-'( {kk& ahhG  L1c( 77emm #  ( (WS\2::fCUWYW_W_`a`g`g`iWj k WW %  ( (WS\2::fCUWYW_W_`a`g`g`iWj k%(@ &JK KLsS=()nnQ]388:==ahhG#h JLLs DHHceZdZdZdZeDcgc]}d| c}}ZgdZ ddZdZe de e e fd e jd dfd Zdd Zd ZdZycc}}w) QuantStatezWcontainer for quantization state components to work with Params4bit and similar classes)fp4nf4bitsandbytes__) absmax quant_map nested_absmaxnested_quant_map quant_state quant_type blocksizerZr\nested_blocksize nested_dtype nested_offsetNc ||_||_||_||_||_||_||_||_|du|_yr#) rr\rrZr r rstate2nested) rrr\rr r rZrrs rrzQuantState.__init__5sH    "$  D( r!cB|jrU|j|j|j|j|j |j g|jg}||S|j|j|j|jd|jg}||S)a$ ensures compatibility with older quant state scheme with nested lists. assumes the following layout: state = [qabsmax, input_shape, A.dtype, blocksize, [offset, state2], quant_type] state2 = [absmax, input_shape, A.dtype, blocksize, None, quant_type] N)rrr\rZr rrr )rr list_reprs r __getitem__zQuantState.__getitem__Js ;;   dkk* I~djj$**dnndTXTcTcdI~r!qs_dictrIrc |jDcgc]'\}}d|vs t|tjs&|)}}}t |sd|vr t dt |dk7s#|dj dd|jvrt d|jd |dt |dk(r.|d}|jt|j||jDcic]\}}|j dd|}}}t|jj|jsJd |vrutjt!|d j#|}||d j#||d |d j#|t%t|d}nd\}}||d|dj#||d|dj#|t%t|d|dtj&|dnd||} | Scc}}wcc}}w)aO unpacks components of state_dict into QuantState where necessary, convert into strings, torch.dtype, ints, etc. qs_dict: based on state_dict, with only relevant keys, striped of prefixes. item with key `quant_state.bitsandbytes__[nf4/fp4]` may contain minor and non-tensor quant state items. rr z"++C,=,=>>> g %\\%(@"ABEEfMF/226:!"45/033F;eW^%<= F(NFF|,8$''/k*%((0%!1229'2B2N%**WW-.TX  KgDs H7H7H79H=c|j|j|j|jt |j j dt|jd}|jr|j|jj|jj|jjjt |jj j d|jjd|s|S|jDcic]#\}}t!|t"j$s!||%}}}|jDcic]#\}}t!|t"j$r!||%}}}t'||d|jz<|Scc}}wcc}}w)z returns dict of tensors and strings to use in serialization via _save_to_state_dict() param: packed -- returns dict[str, torch.Tensor] for state_dict fit for safetensors saving ztorch.)r rr rrZr\)rr rr r zquant_state.bitsandbytes__)r rr rstrrZstriptupler\rrrcloneritemrrrBrr )rpackedrr$rqs_packed_dictnon_tensor_dicts ras_dictzQuantState.as_dictsT //kk_**844::&   ;; NN%)[[%7%7(, (=(=(, (8(8(>(>(@$' (9(9$:$@$@$J%)[[%5%5%7  N,3==?Z41ajELL>Y!Q$ZZ,3MMO_DAq:aQVQ]Q]C^1a4__NabqNr84??JK[_s/"F5F5-"F;F;c|jj||_|jj||_|jr|jj||_|j jj||j _|j jj||j _yyr#)rrrrrr)rrIs rrz QuantState.tosIILL( kknnV, ;;++..0DK!%!3!3!6!6v!>DKK #{{//226:DKK  r!ct|tsytj|j|jdxr/|j |j k(xrtj|j |j dxr|j|jk(xr|j|jk(xr|j|jk(xr|j%|j|j|jk(n|j|juxrI|j%|j|j|jk(S|j|juS)NFgư>)atol) rrrBallcloserr\rrZr r rr)rothers r__eq__zQuantState.__eq__s&%, NN4;; 4 @  ekk) tyy%**4@  ekk) %//1   5#3#33  ;;*u||/G u||+[[ELL0 ;;*u||/G u||+ [[ELL0 r!)NNNNNNNr4)r5r6r7__doc__valid_quant_typesrr!rrr8dictr)rrBrIr'r1rr7).0xs00rrr!sa&8IJ1N1#.J M$ )*(0S#X0 000d@; kKs Arrrc|;dtvr*tj|jtd<td}tj j jj||j|j|\}}|r]|j}||z}t||d\} } t| |j|jd||j|| } n4t||j|jd||j} ||j|n|}| |j| j| _ || fS)aWQuantize a tensor in blocks of values. The input tensor is quantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is calculated for scaling the non-linear quantization. Args: A (`torch.Tensor`): The input tensor. Supports `float16`, `bfloat16`, or `float32` datatypes. code (`torch.Tensor`, *optional*): A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type. For more details, see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]. absmax (`torch.Tensor`, *optional*): A tensor to use to store the absmax values. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 4096. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. nested (`bool`, *optional*): Whether to additionally quantize the absmax values. Defaults to False. Raises: ValueError: Raised when the input data type is not supported. Returns: `Tuple[torch.Tensor, QuantState]`: A tuple containing the quantization results. - `torch.Tensor`: The quantized tensor. - [`QuantState`]: The state object used to undo the quantization. dynamicF)r rT)copy)rrr rZrrrrr rZ) name2qmaprrrIrBops bitsandbytesquantize_blockwisedefaultmeanrrZcopy_r) rqrrror r_out_absmaxrqabsmaxrrs rrDrDs:F | I %#5#7#:#:188#DIi #II**==EE  MD' 6,W RWX -''  !dggahhTg6R^gopovovw !_#))D/$C#\\+*<*<=   r!rr cv||J|=|;dtvr*tj|jtd<td}|t |||t j }|j}|jr\t|j|j}||jz }|jt j k7r|j}|rt jjjj!|||j"j|j|j$|j||St jjjj'|||j"j|j|j$|jS)aDequantize a tensor in blocks of values. The input tensor is dequantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is used for scaling the non-linear dequantization. Args: A (`torch.Tensor`): The quantized input tensor. quant_state ([`QuantState`], *optional*): The quantization state as returned by [`quantize_blockwise`]. Required if `absmax` is not provided. absmax (`torch.Tensor`, *optional*): A tensor containing the scaling values. Required if `quant_state` is not provided and ignored otherwise. code (`torch.Tensor`, *optional*): A mapping describing the low-bit data type. Defaults to a signed 8-bit dynamic type. For more details, see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561]. Ignored when `quant_state` is provided. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 4096. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. Ignored when `quant_state` is provided. Raises: ValueError: Raised when the input data type is not supported. Returns: `torch.Tensor`: The dequantized tensor. The datatype is indicated by `quant_state.dtype` and defaults to `torch.float32`. r>r@ro)rArrrIrrBr}rrdequantize_blockwiserrrZr"rBrCrorr rE)rqrrrror rs rrMrMswR  "f&88 8 | + I %#5#7#:#:188#DIi # TYV[VcVcd   F%k&8&8+:L:LM+$$$ <<5== (\\^F  3377       )  ! !    8  99 ! ! 6 6 > > AHH%  r!@cd|d}d}|dk(r gd}n4|dk(rgd}n*|dk(rgd}n |dk(r|d k(r gd ddd }n td |td |dtj||}|j|j j |j dk(sJ|S)NrCr)rg 6GgfgTFٿgI4ҿg০ǿg Org__?g`\?g?g@g?g4?g`?g`v"?rr)rg?g @g(@g@g@rg@rgg g(ggggint4)rr rrr/iiaf4rN)rg|8geg:Kڞ׿gH2퓊cпg}Yu-ÿgQ #(DrgF?g`_?g 0E?gL_߹E?gƶ=?ga@?gкv -?rr/z94-bit AbnormalFloats currently only support blocksize 64.z Typename z not supportedrI)rrBrdiv_rrr)typenamerIr rs r get_4bit_typer_fs ~ D5  $ U l V G U  ?"d#D&&&ab b |!IhZ~"FGG <<V ,DIIdhhjnn ::<2   Kr!c $t|||||d|SNr quantize_4bitrqrror compress_statistics quant_storages r quantize_fp4rg FC4GP] ^^r!c $t|||||d|SNrrbrds r quantize_nf4rkrhr!rc |j}tjjjj ||||\}} t ||j} |rB| j} t| | z d\} } ~ t| ||j|| || | }nt| ||j|| |}||j|n|}| |j|j|_ ||fS)aQuantize tensor A in blocks of 4-bit values. Quantizes tensor A by dividing it into blocks which are independently quantized. Args: A (`torch.Tensor`): The input tensor. Supports `float16`, `bfloat16`, or `float32` datatypes. absmax (`torch.Tensor`, *optional*): A tensor to use to store the absmax values. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 64. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. compress_statistics (`bool`, *optional*): Whether to additionally quantize the absmax values. Defaults to False. quant_type (`str`, *optional*): The data type to use: `nf4` or `fp4`. Defaults to `fp4`. quant_storage (`torch.dtype`, *optional*): The dtype of the tensor used to store the result. Defaults to `torch.uint8`. Raises: ValueError: Raised when the input data type is not supported. Returns: Tuple[`torch.Tensor`, `QuantState`]: A tuple containing the quantization results. - `torch.Tensor`: The quantized tensor with packed 4-bit values. - [`QuantState`]: The state object used to undo the quantization. r[r)r )rr\rZr rr rr)rr\rZr rr )r\rBrBrCrcrEr_rIrFrDrrZrGr)rqrror rer rf input_shaperHrIrrrJrstates rrcrcs@''KII**88@@  MD' AHH 5D,Wv-=M ''!  ''!  !_#))D/$C||ELL1 :r!c"t|||||dSradequantize_4bitrqrrror s rdequantize_fp4rs 1k63 5 IIr!c"t|||||dSrjrprrs rdequantize_nf4rv%rtr!c |+||Jt||j|j||}n |j}|jr\t |j|j }||jz }|jtjk7r|j}|ctjjjj|||j|j |j|j|n`tjjjj#|||j|j |j|j}|jddk(r|j%S|S)aDequantizes a packed 4-bit quantized tensor. The input tensor is dequantized by dividing it into blocks of `blocksize` values. The the absolute maximum value within these blocks is used for scaling the non-linear dequantization. Args: A (`torch.Tensor`): The quantized input tensor. quant_state ([`QuantState`], *optional*): The quantization state as returned by [`quantize_4bit`]. Required if `absmax` is not provided. absmax (`torch.Tensor`, *optional*): A tensor containing the scaling values. Required if `quant_state` is not provided and ignored otherwise. out (`torch.Tensor`, *optional*): A tensor to use to store the result. blocksize (`int`, *optional*): The size of the blocks. Defaults to 64. Valid values are 64, 128, 256, 512, 1024, 2048, and 4096. quant_type (`str`, *optional*): The data type to use: `nf4` or `fp4`. Defaults to `fp4`. Raises: ValueError: Raised when the input data type or blocksize is not supported. Returns: `torch.Tensor`: The dequantized tensor. )rr\rZr r rLrr )rr\rZrrrMrrrBr}r"rBrCrqror r rEr2)rqrrror r s rrqrq/sSD!co55 ))))!  ##%k&8&8+:L:LM+$$$ <<5== (\\^F  ..22 v{,,k.D.DkFWFWYdYjYjps 3 ii$$44<<   ! !  " "         wwqzQuuw Jr!c|Vdtvr*tj|jtd<td}|j|j}t j |j }|jtjk7r|j}||z }t|||}|||ffS)Nr>) rArrrIrBrrrZr}r"quantize_no_absmax)rqrrorinps rquantizer{xs  | I %#5#7#:#:188#DIi #wwqxx  YYq\   F ||u}}$ f*C S$ ,C  r!rnc||J|X|Vdtvr*tj|jtd<td}|j|j}|||f}t ||d|}||dzS)Nr>r r)rArrrIdequantize_no_absmax)rqrnrrros r dequantizer~s   22 2 | I %#5#7#:#:188#DIi #wwqxx  } q%(C 0C q>r!c Xt|5|%tj|tj}t ||gt j t|t|t|tj|jddd|S#1swY|SxYw)a Quantizes input tensor to 8-bit. Quantizes the 32-bit input tensor `A` to the 8-bit output tensor `out` using the quantization map `code`. Parameters ---------- A : torch.Tensor The input tensor. code : torch.Tensor The quantization map. out : torch.Tensor, optional The output tensor. Needs to be of type byte. Returns ------- torch.Tensor: Quantized 8-bit tensor. Nr) rUrB zeros_likerrr cquantizertrFrcr)rqrros rryrys,  T ;""1EKK8C1c( gdmWQZrxx ?RS T J T Js B BB)c rt|5|%tj|tj}t |||gt |}t jt|t|t|tj|j|ddd|S#1swY|SxYw)a Dequantizes the 8-bit tensor to 32-bit. Dequantizes the 8-bit tensor `A` to the 32-bit tensor `out` via the quantization map `code`. Parameters ---------- A : torch.Tensor The 8-bit input tensor. code : torch.Tensor The quantization map. out : torch.Tensor The 32-bit output tensor. Returns ------- torch.Tensor: 32-bit output tensor. Nr) rUrBrr}rrr cdequantizertrFrcr)rqrrostreams rr}r}s,  ^ ;""1EMM:C4C.!#A&  wqz73<!'')ATV\] ^ J^ Js BB,,B6optimizer_namegrstate1beta1epsrlrrbeta2beta3alpha weight_decay gnorm_scale unorm_vec max_unormcd}|dkDr-tj|jj}d}|jtj k(r t |d}n|jtjk(r t |d}nd|jtjk(r"tt |dk(r t |d}n%td|jd|jt|||||gt|5|t|t|t|t|t|tj|tj|tj|tj| tj| tj| tj|tj| tj |tj|tj| tj"|tj |j%dddy#1swYyxYw) az Performs an inplace optimizer update with one or two optimizer states. Universal optimizer update for 32-bit state and 32/16-bit gradients/weights. Parameters ---------- optimizer_name : str The name of the optimizer: {adam}. g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Optimizer state 1. beta1 : float Optimizer beta1. eps : float Optimizer epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. state2 : torch.Tensor Optimizer state 2. beta2 : float Optimizer beta2. beta3 : float Optimizer beta3. alpha : float Optimizer alpha. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. skip_zeros : bool Whether to skip zero-valued gradients or not (default: False). rNrr rUrAGradient+optimizer bit data type combination not supported: grad , optimizer )rBrrr"rZr}str2optimizer32bitrbfloat16rrrrUrtrFrrvc_boolr)rrrrrrrrrrrrrrrr skip_zeros param_norm optim_funcs roptimizer_update_32bitrs|J3ZZ / Jww%--'7: EMM !'7: ENN "s+=n+M'NRS'S'7: OPQPWPWyXdekeqeqdr s  q!VVY/0    AJ AJ FO FO I  JJy ! JJz " JJu  JJu  JJu  JJu  JJsO JJ| $ JJt  JJrN JJ{ # IIj ! JJqwwy !%    s E II(zyThis function is deprecated and will be removed in a future release. Please use optimizer_update_8bit_blockwise instead. qmap1qmap2max1max2new_max1new_max2chd}|dkDr-tj|jj}t |5t |||||| | | | ||g |j tjk(rt|j tjk(rVt|dt|t|t|t|t|tj|tj|tj|tj|tj|tj|tj| t| t| t| t| t|t|tj|tj|tj|jn|j tjk(rs|j tjk(rUt|dt|t|t|t|t|tj|tj|tj|tj|tj|tj|tj| t| t| t| t| t|t|tj|tj|tj|jn%t!d|j d|j dddy#1swYyxYw)a Performs an inplace Adam update. Universal Adam update for 32/8-bit state and 32/16-bit gradients/weights. Uses AdamW formulation if weight decay > 0.0. Parameters ---------- optimizer_name : str The name of the optimizer. Choices {adam, momentum} g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Adam state 1. state2 : torch.Tensor Adam state 2. beta1 : float Adam beta1. beta2 : float Adam beta2. eps : float Adam epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. qmap1 : torch.Tensor Quantization map for first Adam state. qmap2 : torch.Tensor Quantization map for second Adam state. max1 : torch.Tensor Max value for first Adam state update. max2 : torch.Tensor Max value for second Adam state update. new_max1 : torch.Tensor Max value for the next Adam update of the first state. new_max2 : torch.Tensor Max value for the next Adam update of the second state. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. rrr rrN)rBrrr"rUrrZr}rstr2optimizer8bitrtrFrrvrrr)rrrrrrrrrrrrrrrrrrrrrs roptimizer_update_8bitrGsZJ3ZZ /  51aE5$hX`ab 77emm #  (C n -a 0   " 9% :& 5! 5! 3 4  2  !! <( ;' 1779%+ .WW %&,,%++*E n -a 0   " 9% :& 5! 5! 3 4  2  !! <( ;' 1779%+ 0STUT[T[S\\hioiuiuhvw g555s MN((N1absmax1absmax2cd}|jtjk(r*|jtjk(r t|d}n|jtj k(r*|jtjk(r t|d}n|jtj k(r?|jtjk(r"tt|dk(r t|d}n%td|jd|jt||||| | ||gt|5|t|t|t|t|tj|tj|tj|tj|tj| tj| tj| t| t| t|t|tj|tj|tj|tj|j!dddy#1swYyxYw)Nrr rUrrr)rZrBr}rstr2optimizer8bit_blockwiserrrrrrUrtrFrrvrr)rrrrrrrrrrrrrrrrrrrrs roptimizer_update_8bit_blockwisers*Jww%--FLLEKK$?0@C EMM !fllekk&A0@C  5>>! LLEKK ' +N; < A0@C OPQPWPWyXdekeqeqdr s  q!VVUE7GDE    AJ AJ FO FO JJu  JJu  JJu  JJu  JJsO JJt  JJrN EN EN G  G  JJ| $ JJ{ # IIj ! JJqwwy !'    s 0EI;;Jgrad gnorm_vec percentilec 6t|5t||g|jtjk(r_t j t|t|tj|tj|jn|jtjk(r_t jt|t|tj|tj|jntd|jddddtj||dz}tj|\}}tj||}d}||kDr||z }|||fS#1swYexYw)aApplies percentile clipping grad: torch.Tensor The gradient tensor. gnorm_vec: torch.Tensor Vector of gradient norms. 100 elements expected. step: int The current optimization steps (number of past gradient norms). zGradient type z not supported!Ndr)rUrrZrBr}r cpercentile_clipping_g32rtrFrvrrcpercentile_clipping_g16rsqrtr) rrrr current_gnormvalsr clip_valuers rpercentile_clippingrsF  K4#$ :: &  ( (  " 4  4::<(   ZZ5== (  ( (  " 4  4::<(  ~djj\IJ J#K&JJy45M 9%ID#D,-JKz! =0 *k 117KKs DFF histogramindex1index2sourcec,t|jdk(sJ|jtjk(sJ|jtjk(sJ|jtj k(sJ|jtj k(sJ|j jdk(sJ|j jdk(sJ|j jdk(sJ|j jdk(sJtj|jd}tj|j}t||||gtjt|t|t|t|||y)NrrCr)rr\rZrBr}int32rItyperFrvrrr chistogram_scatter_add_2drt)rrrrmaxdim1ns rhistogram_scatter_add_2dr:s= y 1 $$ $ ??emm ++ + <<5== (( ( <<5;; && & <<5;; && &    F ** * ==   '' ' ==   '' ' ==   '' 'jj+,G 6<<>"A y&&&12!!')"4gfowvX_`fXgiprstr!c tjjstjj|j|k7s|j|k7r%t d|jd|j|j }|j }|}|} d} t|dk(rt|dk(r|s%| s#|j d|j dk7rd} n|r%| s#|j d|j dk7rd} n|r%| r#|j d|j dk7rd} n|s| r|j d|j dk7rld} nht|dk(rt|dk(r|s%| s#|j d|j dk7rd} n%|r$| s"|j d|j dk7rd} n|r$| r"|j d|j dk7rd} n|s| r|j d|j dk7rd} nt|dk(rt|dk(r|s$| s"|j d|j dk7rd} nq|r$| s"|j d|j dk7rd} nK|r$| r"|j d|j dk7rd} n%|s#| r!|j d|j dk7rd} |a|j } | syt|dk(rjt|dk(r[| d|dk(rO| d|dk(rC|d|dk(r7|d|dk(r+d} n't|dk(rJt|dk(r<|s | s |d|df} n|r | r |d|df} n|r | s |d|df} n|s| r|d|df} nt|dk(rZt|dk(rL|s| s|d|d|df} n|r| r|d|d|df} n|r| s|d|d|df} nz|sx| rv|d|d|df} ngt|dk(rYt|dk(rK|s| s|d|d|df} n8|r| r|d|d|df} n%|r| s|d|d|df} n|s| r|d|d|df} | std |d |d |d | d  S) Nz3Expected torch.int8 input tensors A and B, but got  and Trr rFrUz?Tensor dimensions incorrect for matrix mulitiplication: A x B:  x z with transpose for A x B: r) rBrCis_initializedinitrZ TypeErrorr\rr) rqrro transposed_A transposed_B expected_typesAsBtAtBcorrectsouts r check_matmulrMsr :: $ $ & ww-177m#;MaggYV[\]\c\c[deff B B B BG 2w!|B1 "qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G RA#b'Q,"qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G RA#b'Q,"qwwqz!9G qwwqzQWWQZ7G B1771:3GqwwqzQWWQZ7G yy3r7aio supported for tensor contractions, but dims for A x B were: r)rrBrrrIrr\ batched_igemmstriderr;r,rHrr cigemmrFrrvrt)rqrrorrrrrrldbmr$ldaldcptrs rigemmrs 1c< >D {kkt5;;qxxH 177|qS\Q. 771: # aggaj(@ As+ + B BB1 eRU^ #b'Q,eRUBqE "B1 eRU^ #b'Q,eRUBqE " 2w!| 88:a=AGGAJ & L XXZ]aggaj (L qww<1 xxz!} *$ A!''!*,# xxz!} *$ A!''!*,# r7a<1A((*,QA6C W\c"gl11 AQ%C qE qEhhj|!4e RA2w!||1A2a5BqE>_`b_ccfgifjk   qE qE qEBqEMe  % % ' 3 3AHH =Cq!SkJJ  , , 1  1  1     3 3 3  Jr!ct|jdk(rt|jdk(s%td|jd|jt|||||}|0t j |tj |j}|jr|jd}d}n|j}|d|jdk7r$|j}|jd}n|d|jdk(rd }|jd}n{|ddk(r$|j}|jd}nO|ddk(r$|j}|jd}n#|j}|jd}|jr|jd}d}n|j}|d|jdk7r&|j}|jd}d}nP|d|jdk(r|jd}d }n%|j}|jd}d}|jd} |jd} |jd} |jd} | } |jd|jdz}|jd|jdz}|jd|jdz}tjj|j}t|||gtj |t#j$|t#j$|t#j&| t#j&| t#j&| t)|t)|t)|t#j&|t#j&|t#j&| t#j*|t#j*|t#j*|t#j,| |S) NrUz@Expected 3-dimensional tensors for bmm, but got shapes A and B: rrr FrrT)rr\rrrBrrrI is_contiguousr contiguousr;r,rHrr cbatched_igemmrFrrvrtc_longc_uint32)rqrrorrrrsr num_batchrrr$rstrideAstrideBstrideCrs rrr+so qww<1 CLA$5[\]\c\c[ddijkjqjqirstt 1c< >D {kkt5;;qxxHhhjm HHJ Q41771:  A((*Q-C qTQWWQZ L((*Q-CtqyLLNhhjm1LLNhhjmLLNhhjmhhjm HHJ Q41771:  A((*Q-C L qTQWWQZ ((*Q-CL A((*Q-C L I  A  A  A Cggaj1771:%Gggaj1771:%Gggaj1771:%G  % % ' 3 3AHH =C q!Sk  , , 1  1  1     3 3 3 ' ' ' I!$ Jr!c|7tjjjj ||||Stjjjj ||S)aLPerforms an 8-bit integer matrix multiplication. A linear transformation is applied such that `out = A @ B.T`. When possible, integer tensor core hardware is utilized to accelerate the operation. Args: A (`torch.Tensor`): The first matrix operand with the data type `torch.int8`. B (`torch.Tensor`): The second matrix operand with the data type `torch.int8`. out (`torch.Tensor`, *optional*): A pre-allocated tensor used to store the result. dtype (`torch.dtype`, *optional*): The expected data type of the output. Defaults to `torch.int32`. Raises: `NotImplementedError`: The operation is not supported in the current environment. `RuntimeError`: Raised when the cannot be completed for any other reason. Returns: `torch.Tensor`: The result of the operation. )rBrBrCint8_linear_matmulrorE)rqrrorZs rrrsS&  1155aC@ 99 ! ! 4 4 < This function is useful for training, but for inference it is advised to use [`get_row_absmax`] instead. The column-wise quantization scales are not typically needed in inference scenarios. Args: A (`torch.Tensor` with dtype `torch.float16`): Input tensor. row_stats (`torch.Tensor`, *optional*): If provided, calculation of row statistics is skipped. col_stats (`torch.Tensor`, *optional*): If provided, calculation of column statistics is skipped. nnz_block_ptr (`torch.Tensor`, *optional*): Not used. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing quantization statistics. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization statistics. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization statistics. - `torch.Tensor` with dtype `torch.bool`, *optional*: A mask indicating the locations of outliers in the input tensor. Nr/rrFdimkeepdim)is_floating_pointrrhr\ masked_fill_get_row_absmaxamaxr")rqrrr threshold outlier_maskabsAs rget_colrow_absmaxrsD    LI-uuw||B , s?9,L   lC 0  'q)4I   a 7==?I i --r!c *|jtjk(sJt|jdd}|jd}tj |ftj |j}t|gt|5tjt|t|tj|tj|tj|t!|ddd|S#1swY|SxYw)aTDetermine the quantization statistics for input matrix `A` in accordance to the `LLM.int8()` algorithm. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `torch.Tensor` with dtype `torch.float32`: The absolute maximum value for each row, with outliers ignored. Nr/rY)rZrBrrr\emptyr}rIrrUr cget_row_statsrtrFrrvr)rqrrowscolsrs rrrs 77emm ## #  D 772;D TG5==JI qcN     AJ I  JJy ! JJt  JJt  q !      s A/DDc heZdZdedededej dej dej f dZy) COOSparseTensorrrnnzrowidxcolidxrc|jtjk(sJ|jtjk(sJ|jtjk(sJ|j |k(sJ|j |k(sJ|j |k(sJ||_||_||_||_||_ ||_ yr#) rZrBrrrrrrrrr)rrrrrrrs rrzCOOSparseTensor.__init__"s||u{{***||u{{***||u}},,,||~$$$||~$$$||~$$$     r!N)r5r6r7rrBrrr9r!rrr!sE"),6;llLQLLbgbnbnr!rceZdZdZy)CSRSparseTensorc|jtjk(sJ|jtjk(sJ|jtjk(sJ|j |k(sJ|j |k(sJ|j |dzk(sJ||_||_||_||_||_ ||_ yNr ) rZrBrrrrrrrowptrrr)rrrrr rrs rrzCSRSparseTensor.__init__5||u{{***||u{{***||u}},,,||~$$$||~$$$||~)))     r!Nr5r6r7rr9r!rr r 4 r!r ceZdZdZy)CSCSparseTensorc|jtjk(sJ|jtjk(sJ|jtjk(sJ|j |k(sJ|j |k(sJ|j |dzk(sJ||_||_||_||_||_ ||_ yr ) rZrBrrrrrrcolptrrr)rrrrrrrs rrzCSCSparseTensor.__init__Frr!Nrr9r!rrrErr!rctj|jd\}}|jdtj|j dzftj |jj}|j|j|jd|jdt|j |j|j||j|j SNT return_countsr rYr)rAsrcr)rBuniqueradd_rrrrIscatter_rrcumsum_r rrrr)cooArcountsr s rcoo2csrr Vs\\$++TBNFF KKN [[$))a-)T[[EWEW XF OO&++-VZZ\qOA NN1 499dii64;;PTP[P[ \\r!cFtj|j\}}|j|}|j|}tj |d\}}|j dtj|jdzftj|jj}|j|j|jd|jdt|j |j|j"|||Sr)rBrrrrrrrrrrIrrrrrrr)rr col2rowidxrr colvaluesrrs rcoo2cscr$_sjj-OC [[ $F [[ $F S=Iv NN1 [[$))a-)T[[EWEW XF OO)..* !OD NN1 499dii666 RRr!ctj|ftj|}tj|ftj|}tj|f||}t||||||S)NrY)rBrrr)rrrrIrZrrrs r coo_zerosr&ks[ [[#u{{6 BF [[#u{{6 BF [[#uV This function is useful for training, but for inference it is advised to use [`int8_vectorwise_quant`] instead. This implementation performs additional column-wise transposed calculations which are not optimized. Args: A (`torch.Tensor` with dtype `torch.float16`): The input matrix. col_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantization scales. row_stats (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantization scales. out_col (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the column-wise quantized data. out_row (`torch.Tensor`, *optional*): A pre-allocated tensor to hold the row-wise quantized data. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The row-wise quantized data. - `torch.Tensor` with dtype `torch.int8`: The column-wise quantized data. - `torch.Tensor` with dtype `torch.float32`: The row-wise quantization scales. - `torch.Tensor` with dtype `torch.float32`: The column-wise quantization scales. - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features. zUrow_stats must be None. int8_double_quant() does not support pre-allocated row_stats.zUcol_stats must be None. int8_double_quant() does not support pre-allocated col_stats.zQout_col must be None. int8_double_quant() does not support pre-allocated out_col.zQout_row must be None. int8_double_quant() does not support pre-allocated out_row.)r)rrBrBrCint8_double_quantrE)rqrrr'r(rs rr*r*rssNpqqpqqlmmlmm 99 ! ! 3 3 ; ;A ; SSr!statscjtjjjj ||S)aYDequantizes a tensor with dtype `torch.int8` to `torch.float32`. Args: A (`torch.Tensor` with dtype `torch.int8`): The quantized int8 tensor. stats (`torch.Tensor` with dtype `torch.float32`): The row-wise quantization statistics. Returns: `torch.Tensor` with dtype `torch.float32`: The dequantized tensor. )rBrBrCint8_vectorwise_dequantrE)rqr+s rr-r-s' 99 ! ! 9 9 A A!U KKr!cjtjjjj ||S)awQuantizes a tensor with dtype `torch.float16` to `torch.int8` in accordance to the `LLM.int8()` algorithm. For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339). Args: A (`torch.Tensor` with dtype `torch.float16`): The input tensor. threshold (`float`, *optional*): An optional threshold for sparse decomposition of outlier features. No outliers are held back when 0.0. Defaults to 0.0. Returns: `Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]`: A tuple containing the quantized tensor and relevant statistics. - `torch.Tensor` with dtype `torch.int8`: The quantized data. - `torch.Tensor` with dtype `torch.float32`: The quantization scales. - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features. )rBrBrCint8_vectorwise_quantrE)rqrs rr/r/s'$ 99 ! ! 7 7 ? ?9 MMr!rct|ts|jr|jtj k(sJdt|j d|j d|j|jdj|jdj|j}|Et j|j|j df|j|j}|j}|j j#|k(sJ|j$j#|k(sJ|jj#|k(sJ|j&|j dk(sJ|j)rdnd}|j+|rdnd}|j d}t,j/j0}t3|j }t3|j$} t3|j} t3|} t3|} t5j6|j} t5j6|j}t5j6|j&}t5j6|j d}t5j6|}t5j6|}t9|j |j$|j||gt;j<||| | | ||||| || t5j>| |S)Nz8Tensor must be `COOSparseTensor or a PyTorch COO tensor.rr )rrrrrrrIrZFT) rr is_sparselayoutrB sparse_coor\_nnzindicesrrrrrIrZrrrrrrrrLr,r>rtrFrvrr cspmm_coor)rrrorrrrr ptrRowidx ptrColidx ptrValuesptrBptrCcnnzcrowsAccolsAccolsBcldbcldcs rspmm_coorCs dO ,~~$++1A1A"A F A AA <<>!$((*<<>!$((*;;=   {kk499aggaj1!((!''R ((C ;;   # %% % ;;   # %% % ;;   # %% % 99 "" "OO-54L ((*MM       , Jr!cd|Otj|j|jdf|j|j j }|j}|jj|k(sJ|jj|k(sJ|j j|k(sJ|j|jdk(s J|jd|j|jrdnd}|j|rdnd}|jd}tj|jd\}} | jdj!} tj"| d\} } | j!} | j!} | dd ksJd | dd |j tj$tj&fvsJt)| } t)| }t)| }t)|j}t)|j}t)|j }t)|}t)|}t)|}t+j,| j}t+j,|j}t+j,|j}t+j,|j}t+j,|jd}t+j,|jd}t+j,|}t+j,|}t/|5t1|j|j|j |||g|j tj$k(r#t3j4||| |||||||||||n?|j tj&k(r"t3j6||| |||||||||||ddd|S#1swY|SxYw) Nr r1rz vs FTr) descending z)Current max count per row is 8 but found r)rBrrr\rIrrZrrrrrrrrcumsumrrrint8rtrFrvrUrr cspmm_coo_very_sparse_naive_fp16 cspmm_coo_very_sparse_naive_int8)rr dequant_statsrorrrrrrr max_countmax_idx ptrOffset ptrMaxCount ptrMaxIdxr8r9r:r;r<ptrDequantStats cnnz_rowsr=r>r?crowsBr@rArBs rspmm_coo_very_sparserT sd {kk499aggaj1!(($++J[J[\ ((C ;;   # %% % ;;   # %% % ;;   # %% % 99 "?tyykaggY$?? "OO-54L ((**I ::dhh D ZZ "F ZZ "F ZZ #F ZZ #F ::c?D ::c?D  #4;; T[[!S-PQ 77emm #  0 0  WW "  0 0 )#L JM#L Js ,B/P%%P/g_@ztThis function is deprecated and will be removed in a future release. Consider using `int8_vectorwise_quant` instead.vectorc(|dk(rmtj|jj}tj||z dzj tj }||fS|dvrktjtj||d}tj|t|z zj tj }||fS|dk(r|j}|j}|j|jz }|dk(rd}d |z }|j}tj||z} tj||z| z | z}||fS|d vr|j}|j}tj||dtj||dz }d||dk(<d |z }tj||d}tj||z} tj||z| z | z}||fS|d k(rtj5tj|} tj| |d}|d z}| |j| kD} tj|| } |j| | | z|| <tj||z tzj tj }ddd||fSy#1swYfSxYw) Nlinear)rUrowTr zeropointrr go@)vector-zeropoint row-zeropointtruncated-vectorgffffff?)rBrrr"rrrHrCrZminaminno_grad expand_asr) r<rr rxqrZdynaqxminxzpxabsxrrs rvectorwise_quantri` s Xyy|!'') [[TC ( + +EJJ 74x ( (zz%))A,C> [[a$h ( + +EJJ 74x { " GGIuuw  19D T\uuwkk$)$ KKQ % +"u < < GGIzz!d3ejjUY6ZZTQY T\zz!d3kk$)$ KKQ % +"u ) ) ]]_ :99Qrs$ %'' (I             "" "" !! !!           !! !!           $$ $$ $$9!J %% %%  )) )) (( (( %% %%  %% %%  )) )) +: ** ** **  .. .. .. -- -- -- ** ** **  -- -- -- .. .. ..5D''0**2"!ELLq1  ::q '5<<'(5<<("MM2CKu||K!6-&S&:(V43n R]jk  l  !x 67!HPvP"++P %x %HR[[$9 % R]jk#' @ @ %,, @ @  @l@Fn n f$(%)"&  D ||D 5<< D U\\ "D %,,  D 5<< #$DR)-%)#'"& J ||J*%J U\\ "J 5<< J %,,  J  J \\JZOh&*"&++ _ ||_ U\\ "_ %,, _&*"&++ _ ||_ U\\ "_ %,, _&*"&++J ||J U\\ "J %,, J 5<< #$J^)-%)"& J ||J*%J U\\ "J %,,  J  J  \\ J)-%)"& J ||J*%J U\\ "J %,,  J  J  \\ J)-%)"& F ||F*%F U\\ "F %,,  F  F \\FR R]jk$("&  5<<  %,,  65( () l& R]jk.2%)#'"&   E&&.) * U\\ " 5<<  %,,    l( R]jk&Xell5KW]l< R]jkF&x 7MY_lP&*(,#d d  d d   d  d  d  d  d  U\\ "d  d  d  d d d  %d !d $ %d N ; ,(,)AA AA  A U\\ " A  A A A A A A ELL !A A 5<< AA u||$!A"#A$%A& %'A()A* +A  Aj'= =  = =   = U\\ " =  =  =  =  =  =  =  =  =  ELL != = ell #!= "#= $%= ( )= @ R]jk&2f&2&2s&2PS&2l&2R R]jkuuuuX^ulu$GLjjQn#'  " " " %,, "P#' b b b %,, bP#' ] ] ] %,, ]@X\chcncnC%,,C5<<Chu||>TC<#'#'  |||||| %,,   5<< 8 R]jk)-(,,0 6. ||6. %6. %6.ELL) 6.  5<<x '= => 6.l6.r R]jk"ell"l"J&""] S.3ZZD)-(,&*&* 0T ||0T %0T %0Tell # 0T ell # 0Tf Lu|| LELL LNU\\N0#'@ - .@ ||@ %,, @FOd  6  )  )X J -2JJ88 8r!