UhxUdZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z ddl mZddlmZmZddlmZdd lmZdd lmZdd lmZmZmZmZmZmZd d lm Z ddl!m"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(erddl)m*Z*ndZ*ejVe,Z-ereZ.ee/e e e/e e/ffe0d< neegderdnderdndffdderdndffdderdndffdderdndffdderdndffdd erd!nderd"ndffd#d$derdndffd%erd&nddffd'd(d)erd*nderd+ndffd,d-erd.ndffd/d0derd1ndffd2d3d4derdndffd5d6erd7ndffd8derd9ndffd:d;erdd?erd@nderdAndffdBdCerdnderdndffdDderdndffdEd;erdnddffd?d@dAerdBndffdCderdwndffdDd6erd7ndffdEerdnderdndffdFerdnderdndffdGdHdIdJderdndffdKerdLnderdMndffdNerdnderdndffdOderdndffdPderdndffdQderdndffdRderdndffdSdTdUdVdWdXdYerdZndffd[dGerdHndffd\erd]nderd^ndffd_d`erdanddffdberdnderdndffdcerdnderdndffdderdenderdfndffdgerdnderdndffdherdnderdndffdierdnderdndffdjerdnderdndffZ.e"e$e.Z1e$jdDcic]\}}|| c}}Z3dke/fdlZ4 dzdme e/ejjfdne e e/ejjfdoe6dpe e6dqe e e/e/fdre e e6e/fdse e/dte6due/fdvZ7GdwdxZ8dydxgZ9ycc}}w({zAuto Tokenizer class.N) OrderedDict) TYPE_CHECKINGDictOptionalTupleUnion)PretrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)load_gguf_checkpoint)PreTrainedTokenizer)TOKENIZER_CONFIG_FILE) cached_fileextract_commit_hashis_g2p_en_availableis_sentencepiece_availableis_tokenizers_availablelogging)EncoderDecoderConfig)_LazyAutoMapping)CONFIG_MAPPING_NAMES AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)PreTrainedTokenizerFastTOKENIZER_MAPPING_NAMESalbertAlbertTokenizerAlbertTokenizerFastalign BertTokenizerBertTokenizerFastariaLlamaTokenizerLlamaTokenizerFast aya_visionCohereTokenizerFastbark)bart) BartTokenizerBartTokenizerFastbarthezBarthezTokenizerBarthezTokenizerFast)bartpho)BartphoTokenizerNbertzbert-generationBertGenerationTokenizer)z bert-japanese)BertJapaneseTokenizerN)bertweet)BertweetTokenizerNbig_birdBigBirdTokenizerBigBirdTokenizerFastbigbird_pegasusPegasusTokenizerPegasusTokenizerFast)biogpt)BioGptTokenizerNbitnetr) blenderbot)BlenderbotTokenizerBlenderbotTokenizerFast)zblenderbot-small)BlenderbotSmallTokenizerNblipzblip-2 GPT2TokenizerGPT2TokenizerFastbloomBloomTokenizerFast bridgetowerRobertaTokenizerRobertaTokenizerFastbros)byt5) ByT5TokenizerN camembertCamembertTokenizerCamembertTokenizerFast)canine)CanineTokenizerN chameleon chinese_clipclapclip CLIPTokenizerCLIPTokenizerFastclipseg)clvp) ClvpTokenizerN code_llamaCodeLlamaTokenizerCodeLlamaTokenizerFastcodegenCodeGenTokenizerCodeGenTokenizerFastcoherecohere2colpaliconvbertConvBertTokenizerConvBertTokenizerFastcpm CpmTokenizerCpmTokenizerFast)cpmant)CpmAntTokenizerN)ctrl) CTRLTokenizerN)zdata2vec-audioWav2Vec2CTCTokenizerNz data2vec-textdbrxdebertaDebertaTokenizerDebertaTokenizerFastz deberta-v2DebertaV2TokenizerDebertaV2TokenizerFast deepseek_v3 diffllama distilbertDistilBertTokenizerDistilBertTokenizerFastdprDPRQuestionEncoderTokenizerDPRQuestionEncoderTokenizerFastelectraElectraTokenizerElectraTokenizerFastemu3ernieernie_mErnieMTokenizer)esm) EsmTokenizerNfalcon falcon_mambaGPTNeoXTokenizerFastfastspeech2_conformerFastSpeech2ConformerTokenizer)flaubert)FlaubertTokenizerNfnet FNetTokenizerFNetTokenizerFast)fsmt) FSMTTokenizerNfunnelFunnelTokenizerFunnelTokenizerFastgemmaGemmaTokenizerGemmaTokenizerFastgemma2gemma3 gemma3_textgitglmglm4zgpt-sw3GPTSw3Tokenizergpt2 gpt_bigcodegpt_neogpt_neox)gpt_neox_japanese)GPTNeoXJapaneseTokenizerNgptj)zgptsan-japanese)GPTSanJapaneseTokenizerNzgrounding-dinogroupvitheliumherbertHerbertTokenizerHerbertTokenizerFast)hubertrsibertideficsidefics2idefics3 instructblipinstructblipvideointernvlQwen2TokenizerQwen2TokenizerFastjambajanusjetmoe)jukebox)JukeboxTokenizerNzkosmos-2XLMRobertaTokenizerXLMRobertaTokenizerFastlayoutlmLayoutLMTokenizerLayoutLMTokenizerFast layoutlmv2LayoutLMv2TokenizerLayoutLMv2TokenizerFast layoutlmv3LayoutLMv3TokenizerLayoutLMv3TokenizerFast layoutxlmLayoutXLMTokenizerLayoutXLMTokenizerFastled LEDTokenizerLEDTokenizerFastliltllamallama4 llama4_textllava llava_nextllava_next_videollava_onevision longformerLongformerTokenizerLongformerTokenizerFastlongt5 T5TokenizerT5TokenizerFast)luke) LukeTokenizerNlxmertLxmertTokenizerLxmertTokenizerFastm2m_100M2M100Tokenizermambamamba2marianMarianTokenizermbartMBartTokenizerMBartTokenizerFastmbart50MBart50TokenizerMBart50TokenizerFastmegaz megatron-bert)zmgp-str)MgpstrTokenizerNmistralmixtralmllamamlukeMLukeTokenizer mobilebertMobileBertTokenizerMobileBertTokenizerFast modernbert moonshinemoshimpnetMPNetTokenizerMPNetTokenizerFastmptmramt5 MT5TokenizerMT5TokenizerFastmusicgenmusicgen_melodymvp MvpTokenizerMvpTokenizerFast)myt5) MyT5TokenizerNnemotronnezhanllb NllbTokenizerNllbTokenizerFastznllb-moe nystromformerolmoolmo2olmoez omdet-turbo oneformerz openai-gptOpenAIGPTTokenizerOpenAIGPTTokenizerFastoptowlv2owlvit paligemmapegasus pegasus_x) perceiver)PerceiverTokenizerN persimmonphiphi3phimoe)phobert)PhobertTokenizerN pix2structpixtralplbartPLBartTokenizer) prophetnet)ProphetNetTokenizerNqdqbertqwen2 qwen2_5_omni qwen2_5_vl qwen2_audio qwen2_moeqwen2_vlqwen3 qwen3_moe)rag) RagTokenizerNrealmRealmTokenizerRealmTokenizerFastrecurrent_gemmareformerReformerTokenizerReformerTokenizerFastrembertRemBertTokenizerRemBertTokenizerFast retribertRetriBertTokenizerRetriBertTokenizerFastrobertazroberta-prelayernorm)roc_bert)RoCBertTokenizerNroformerRoFormerTokenizerRoFormerTokenizerFastrwkv seamless_m4tSeamlessM4TTokenizerSeamlessM4TTokenizerFastseamless_m4t_v2 shieldgemma2siglipSiglipTokenizersiglip2speech_to_textSpeech2TextTokenizer)speech_to_text_2)Speech2Text2TokenizerNspeecht5SpeechT5Tokenizer)splinter)SplinterTokenizerSplinterTokenizerFast squeezebertSqueezeBertTokenizerSqueezeBertTokenizerFaststablelm starcoder2switch_transformerst5)tapas)TapasTokenizerN)tapex)TapexTokenizerN)z transfo-xl)TransfoXLTokenizerNtvpudop UdopTokenizerUdopTokenizerFastumt5 video_llavaviltvipllava visual_bert)vits) VitsTokenizerN)wav2vec2rs)z wav2vec2-bertrs)zwav2vec2-conformerrs)wav2vec2_phoneme)Wav2Vec2PhonemeCTCTokenizerNwhisperWhisperTokenizerWhisperTokenizerFastxclipxglm XGLMTokenizerXGLMTokenizerFast)xlm) XLMTokenizerNzxlm-prophetnetXLMProphetNetTokenizerz xlm-robertazxlm-roberta-xlxlnetXLNetTokenizerXLNetTokenizerFastxmodyosozambazamba2 class_namec|dk(rtStjD]<\}}||vs t|}t j d|d} t ||cStjjD]"\}}|D]}t |dd|k(s|ccS$t j d}t||r t ||Sy#t$rYwxYw)Nr.ztransformers.models__name__ transformers) rr itemsr importlib import_modulegetattrAttributeErrorTOKENIZER_MAPPING_extra_contenthasattr)r module_name tokenizersmoduleconfig tokenizer main_modules |/var/www/catia.catastroantioquia-mas.com/valormas/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.pytokenizer_class_from_namers..&&#:#@#@#B Z  #3K@K,,q ->@UVF vz22 0>>DDF! # !Iy*d3zA   !!)).9K{J'{J// "  s C CCpretrained_model_name_or_path cache_dirforce_downloadresume_downloadproxiestokenrevisionlocal_files_only subfolderc  | jdd} | )tjdt| t d| }| j dd} t |t||||||||ddd| } | tjdiSt| | } t| d 5} tj| }ddd| d<|S#1swYxYw) a Loads the tokenizer configuration from a pretrained model tokenizer configuration. Args: pretrained_model_name_or_path (`str` or `os.PathLike`): This can be either: - a string, the *model id* of a pretrained model configuration hosted inside a model repo on huggingface.co. - a path to a *directory* containing a configuration file saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the configuration files and override the cached versions if they exist. resume_download: Deprecated and ignored. All downloads are now resumed by default when possible. Will be removed in v5 of Transformers. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, will only try to load the tokenizer configuration from local files. subfolder (`str`, *optional*, defaults to `""`): In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. Passing `token=True` is required when you want to use a private model. Returns: `Dict`: The configuration of the tokenizer. Examples: ```python # Download configuration from huggingface.co and cache. tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased") # This model does not have a tokenizer config so the result will be an empty dict. tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base") # Save a pretrained tokenizer locally and you can reload its config from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") tokenizer.save_pretrained("tokenizer-test") tokenizer_config = get_tokenizer_config("tokenizer-test") ```use_auth_tokenNrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`. _commit_hashF) rrrrrrrr _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errorsrz\Could not locate the tokenizer configuration file, will try to use the model config instead.zutf-8)encoding)popwarningswarn FutureWarning ValueErrorgetrrloggerinforopenjsonload)rrrrrrrrrkwargsr commit_hashresolved_config_filereaderresults rget_tokenizer_configrsRZZ 0$7N!  A    uv v**^T2K&%%'))..305  # rs %&:KHK "W 5#6"#(F> M##s CC cNeZdZdZdZeeedZe ddZ y) AutoTokenizera This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the [`AutoTokenizer.from_pretrained`] class method. This class cannot be instantiated directly using `__init__()` (throws an error). ctd)Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)EnvironmentError)selfs r__init__zAutoTokenizer.__init__1s _  c  |jdd}|>> from transformers import AutoTokenizer >>> # Download vocabulary from huggingface.co and cache. >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> # Download vocabulary from huggingface.co (user-uploaded) and cache. >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*) >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") >>> # Download vocabulary from huggingface.co and define model-specific arguments >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True) ```rNrrrrT _from_autouse_fasttokenizer_typetrust_remote_code gguf_filezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, c3 K|]}|ywN.0cs r z0AutoTokenizer.from_pretrained..s Kq Ks rzt`use_fast` is set to `True` but the tokenizer class does not have a fast version. Falling back to the slow version.zTokenizer class z is not currently imported.rtokenizer_classauto_maprF)return_tensorsFastrr code_revisionz- does not exist or is not currently imported.z The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.zzThis tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed in order to use this tokenizer.z!Unrecognized configuration class z8 to build an AutoTokenizer. Model type should be one of c34K|]}|jywr)rrs rrz0AutoTokenizer.from_pretrained..s4bAQZZ4bsr)(rrrrrrr joinkeysrrwarningfrom_pretrainedr isinstancetuplelistr rr r for_modelrrrtyperr r ospathisdirregister_for_auto_classendswithrdecoderencoder __class__rr)clsrinputsrrrrrrrrtokenizer_class_tupletokenizer_class_nametokenizer_fast_class_nametokenizer_configconfig_tokenizer_classtokenizer_auto_map gguf_path config_dicthas_remote_codehas_local_code class_ref_tokenizer_class_candidate model_typetokenizer_class_pytokenizer_class_fasts rrzAutoTokenizer.from_pretrained7sZ $4d;  % MME zz'4(4 l-F7OHd+#|::j$/$4d;"JJ':DAJJ{D1   %"O$;$?$?PT$U !$, .~.>>qyy K,C,H,H,J KKLAO ?T ; ";,8&?@Y&ZONN=&"; "!1!5!56G!H! ) )*:6 F%5j%A"%5j%A%E%EoW[%\" " )f&67 +,I9 _X^ _I"6yQV"WX`"aK'11@K@F'775IZ^dF&,%;%; "vz*&///Q%+___%E",D8f):: "$ . )*@AMZ,-Cf-LMUYY 6 ?22 ^^F/V 0E0EF  !7Hf7V 4  4#5G5O;+;;[o^dohnoo$: /0@0@/AB++/994bIZI_I_Ia4b+b*ccd f  rNc| | td|t|tr td|t|tr td|=|;t|tr+|j|k7rtd|jd|d|t j vrt |\}}||}||}t j|||f|y) a Register a new tokenizer in this mapping. Args: config_class ([`PretrainedConfig`]): The configuration corresponding to the model to register. slow_tokenizer_class ([`PretrainedTokenizer`], *optional*): The slow tokenizer to register. fast_tokenizer_class ([`PretrainedTokenizerFast`], *optional*): The fast tokenizer to register. NzKYou need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_classz:You passed a fast tokenizer in the `slow_tokenizer_class`.z:You passed a slow tokenizer in the `fast_tokenizer_class`.zThe fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not consistent with the slow tokenizer class you passed (fast tokenizer has z and you passed z!. Fix one of those so they match!)exist_ok)r issubclassrrslow_tokenizer_classrrregister) config_classrfast_tokenizer_classr existing_slow existing_fasts rrzAutoTokenizer.registers ',@,Hjk k  + ;OQh0iYZ Z  + ;OQd0eYZ Z ! ,$0/1HI$99=QQ['<<==MNbMcd!!  ,;; ;+<\+J (M=#+'4$#+'4$""<2FH\1]hp"qr)NNF) r __module__ __qualname____doc__r classmethodrr r staticmethodrrrrrr)sH &'>?\ @\ |)r)rrrr)NFNNNNF):rrrrr collectionsrtypingrrrrrconfiguration_utilsr dynamic_module_utilsr r modeling_gguf_pytorch_utilsr tokenization_utilsrtokenization_utils_baserutilsrrrrrrencoder_decoderr auto_factoryrconfiguration_autorrrrrtokenization_utils_fastr get_loggerrrr str__annotations__rrCONFIG_TO_TYPErPathLikeboolrr__all__)kvs00rr!sa #>>3\?5<3*B"   H %VaUb[eHSM8C=4P.Q)QRb)X )C)E%4-D-F)D X ?V?X(;^bc dX &@W@Y(<_cd eX D;R;T"7Z^_ `X o>U>W':]ab cX  =X *D*F&D.E.G*T X ( 4)X *o>U>W':]ab c+X ,>X>Z!:`dfj k l-X . ?/X 0 61X 4*D*F&D.E.G*T 3X @!3OfOh5Knr s tAX B 2CX D;R;T7Z^_ `EX F OGX H EIX Jo>U>W':]ab cKX L@W@Y)<_cd eMX Nt5L5N1TXY ZOX P/KbKd1Gjno pQX Ro>U>W':]ab cSX T .UX X,F,H(d0G0I,t WX d 2eX h(B(D$$,C,E(4 gX toF]F_/Beij kuX x&.E.G*T wX F#+B+D'$ EX T#+B+D'$ SX ` .aX d,F,H(d0G0I,t cX p+G^G`-Cfjk lqX r7N7P3VZ[ \sX t8O8Q4W[\ ]uX v)CZC\+?bfg hwX x-JaJc/Fimn oyX |&@&BN*A*C& {X H 2IX J .KX L ?MX N1MdMf3Ilpq rOX Po>U>W':]ab cQX R+G^G`-Cfjk lSX V,F,H(d0G0I,t UX d(B(D$$,C,E(4 cX r(B(D$$,C,E(4 qX ~1PgPi3Lost uX B19P9R5X\ AX N+G^G`-Cfjk lOX Po>U>W':]ab cQX R?V?X(;^bc dSX T.H.J*PTVZ[ \UX V ,WX X;R;T7Z^_ `YX Zd>U>W$:]ab c[X ^(4G4I0tUYZ ]X d 6eX fo>U>W':]ab cgX h .iX j)D[D]+@cgh ikX n(B(D$$,C,E(4 mX |(B(D$$,C,E(4 {X J(B(D$$,C,E(4 IX X(B(D$$,C,E(4 WX d_=T=V&9\`a beX fT8O8Q4W[\ ]gX hd9P9R5X\] ^iX j.H.J*PTVZ[ \kX lo>U>W':]ab cmX n_E\E^.Adhi joX pAXAZ*=`de fqX r$:Q:S 6Y]^ _sX t FuX vo>U>W':]ab cwX x CyX zH_Ha1Dgkl m{X |/BYB[+>aef g}X ~;R;T7Z^_ `X @+G^G`-Cfjk lAX B 7CX D)E\E^+Adhi jEX F7N7P3VZ[ \GX H*D[D],@cgh iIX J*D[D],@cgh iKX LoF]F_/Beij kMX N!?KbKd4Gjn"o pOX P*D[D],@cgh iQX T(B(D$$,C,E(4 SX `t5L5N1TXY ZaX d(B(D$$,C,E(4 cX p 4qX t-G-I)t1H1J-PT sX @-JaJc/Fimn oAX B1PgPi3Lost uCX D1PgPi3Lost uEX F/MdMf1Ilpq rGX H^;R;T%7Z^_ `IX J+JaJc-Fimn oKX N(B(D$$,C,E(4 MX \(B(D$$,C,E(4 [X j(B(D$$,C,E(4 iX v'AXAZ)=`de fwX x,F]F_.Beij kyX z "2LcLe4Hko!p q{X |!1KbKd3Gjn o p}X ~1PgPi3Lost uX B%?%AMt)@)B% AX N .OX P)D[D]+@cgh iQX R.H.J*PTVZ[ \SX Tt7N7P3VZ[ \UX V8O8Q4W[\ ]WX X-G-I)tUYZ [YX \(B(D$$,C,E(4 [X j*D*F&D.E.G*T iX v(D[D]*@cgh iwX xG^G`0Cfjk lyX z 3{X ~(B(D$$,C,E(4 }X L (B(D$$,C,E(4 K X X (BYB[*>aef gY X Z +E+G'TSWX Y[ X \ 1PgPi3Lost u] X ^ D?V?X";^bc d_ X ` 4>U>W!:]ab ca X b t:Q:S6Y]^ _c X d 'AXAZ)=`de fe X f T5L5N1TXY Zg X h 'CZC\)?bfg hi X l &@&BN*A*C& k X x ->U>W):]ab cy X z E\E^0Adh i j{ X | ^;R;T%7Z^_ `} X ~ . X @ $=T=V 9\`a bA X B ?V?X(;^bc dC X F 'A'CO+B+D'$ E X T 'A'CO+B+D'$ S X b  )C)E%4-D-F)D a X n d6M6O2UYZ [o X p t7N7P3VZ[ \q X r t7N7P3VZ[ \s X v  9P9R"5X\] u X | ?CZC\,?bfg h} X @ %CZC\'?bfg  X F _=T=V&9\`a bG X H ?V?X(;^bc dI X J @W@Y)<_cd eK X L +E\E^-Adhi jM X P *D*F&D.E.G*T O X ^ *D*F&D.E.G*T ] X j  k X z (B(D$$,C,E(4 y X F 'CZC\)?bfg hG X H &@W@Y(<_cd eI X J (BYB[*>aef gK X L 4M X N M@W@Y+<_cd eO X P U>W':]ab ciX j*D[D],@cgh ikX l_E\E^.Adhi jmX n .oX p 9qX r >sX t CuX v HwX x+G^G`-Cfjk lyX z?V?X(;^bc d{X ~'A'CO+B+D'$ }X J ,KX L