3jvdUdZddlZddlmZddlmZddlmZmZm Z m Z ddl m Z m Z mZmZmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlm Z ddl!m"Z"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.ede ddZ/dZ0dZ1e2hdZ3e2e4e5d<e2hdZ6e2e7e5d<e2hdZ8e2e7e5d<e2hdZ9e2e7e5d<e2hd Z:e2e7e5d!<e6e8e9e:d"Z;eeeAed-eAefd?ZNd*e?d>eAed-eAefd@ZOdAZPd*e?d/e4d-e?dzfdBZQd*e?d>eAed-eAefdCZRd*e?d>eAed-eAefdDZSefdddEdFdGd*e?dHedIe7dJe2e4dzdKe2e4dzdLe4dMe4d-eAefdNZTefdddEdFdGd*e?dHedIe7dJe2e4dzdKe2e4dzdLe4dMe4d-eAefdOZUy)PuPipeline orchestrator — runs all detection stages in sequence. Note: ``from __future__ import annotations`` is intentionally omitted because this module is compiled with mypyc, which does not support PEP 563 string annotations. N)DEFAULT_MAX_BYTES) EncodingEra) BigramProfilehas_model_variantsinfer_languagescore_best_language) _NONE_RESULTDETERMINISTIC_CONFIDENCE HIGH_BYTESDetectionResultPipelineContext) detect_ascii) is_binary) detect_bom)resolve_confusion_groups)detect_escape_encoding) detect_magic)detect_markup_charset)score_candidates)compute_lead_byte_diversitycompute_multibyte_byte_coveragecompute_structural_score) detect_utf8)detect_utf1632_patterns)filter_by_validity)REGISTRY EncodingInfoget_candidatesapplication/octet-stream)encoding confidencelanguage mime_typeg333333?i@> iso8859-1 iso8859-15cp1252_COMMON_LATIN_ENCODINGS>._ISO_8859_10_DISTINGUISHING>r(r)r+r,r-r.r0r1r2r4r5r6r7r8r9r;r<r=r>r?r@rArBrJrS_ISO_8859_14_DISTINGUISHING>rYrZr[r\_WINDOWS_1254_DISTINGUISHING>rCrDrErFrGrHrKr^rZ_HP_ROMAN8_DISTINGUISHING)z iso8859-10z iso8859-14cp1254z hp-roman8_DEMOTION_CANDIDATES> r(r)r,r9_KOI8_T_DISTINGUISHINGcp932cp949)shift_jis_2004euc_kr_MARKUP_SUPERSET_PROMOTIONSdata markup_resultallowedreturnc|j|Stj|j}|||vr|St|} |j |dt}t|t|j|}t|||}||kDr,t||j|j|jS|S#t t f$r|cYSwxYw)aEPromote a markup-declared encoding to its superset when structural evidence supports it. If the declared encoding has a known superset, the superset validates the data, and the superset's structural score is materially better, return a new result using the superset encoding. Otherwise return the original. stricterrors) r rgetrdecodeUnicodeDecodeError LookupErrorr rr r!r"r#)rrr superset_name superset_infoctx base_scoresuperset_scores J/DATA/.local/lib/python3.12/site-packages/chardet/pipeline/orchestrator.py_try_promote_markup_supersetrs%/33M4J4JKM W <]+M M( 3  C)$9O9O0PRUVJ-dM3GN "   $ $  " "  # #     ,sB;;CCr param_namecz||vr)tj|d|dtdtgSt |ddgS)zReturn a low-confidence result for *encoding*, or ``encoding=None`` if filtered out. ``stacklevel=5`` targets the public caller: detect() -> run_pipeline() -> _run_pipeline_core() -> _make_fallback_or_none().  zL is excluded by include_encodings/exclude_encodings; returning encoding=None) stacklevelg?N)r r!r")warningswarn UserWarningr r )r rrs r_make_fallback_or_nonersQw l!H<(K L   ~ X$ N OOc^tj|ytfd|D S)auReturn True if encoding is a demotion candidate with no distinguishing bytes. Checks whether any non-ASCII byte in *data* falls in the set of byte values that decode differently under the given encoding vs iso-8859-1. If none do, the data is equally valid under both encodings and there is no byte-level evidence for preferring the candidate encoding. Fc32K|]}|dkDs |vywN).0bdistinguishings r z!_should_demote..7sA1D1&s  )rqrany)r rrs @r_should_demoter,s2*--h7NAAA AArg?gffffff?valid_candidates.rc4g}|D]}|jrt|||}||j|j<|tkr@|j 0t |t |jdtz |_|j tkrt||||j }||j|j<|tkr|j tk\rt|||}|tkr|j!| t#|S)aEliminate CJK multi-byte candidates that lack genuine multi-byte structure. Four checks are applied in order to each multi-byte candidate: 1. **Structural pair ratio** (valid_pairs / lead_bytes) must be >= ``_CJK_MIN_MB_RATIO``. Catches files with many orphan lead bytes. 2. **Minimum non-ASCII byte count**: the data must contain at least ``_CJK_MIN_NON_ASCII`` bytes > 0x7F. Tiny files with 1-5 high bytes can accidentally form perfect pairs and score 1.0 structurally. 3. **Byte coverage** (non-ASCII bytes in valid multi-byte sequences / total non-ASCII bytes) must be >= ``_CJK_MIN_BYTE_COVERAGE``. Latin text has many high bytes that are NOT consumed by multi-byte pairs; genuine CJK text has nearly all high bytes accounted for. 4. **Lead byte diversity**: the number of distinct lead byte values in valid pairs must be >= ``_CJK_MIN_LEAD_DIVERSITY``. Genuine CJK text draws from a wide repertoire of lead bytes; European false positives cluster in a narrow band (e.g. 0xC0-0xDF for accented Latin). Returns the filtered candidate list. Structural scores are cached in ``ctx.mb_scores`` for reuse in Stage 2b. N)non_ascii_count) is_multibyter mb_scoresname_CJK_MIN_MB_RATIOrlen translater _CJK_MIN_NON_ASCIIr mb_coverage_CJK_MIN_BYTE_COVERAGE_CJK_DIVERSITY_MIN_NON_ASCIIr_CJK_MIN_LEAD_DIVERSITYappendtuple)rrrgatedencmb_score byte_coveragelead_diversitys r_gate_cjk_candidatesrXs:!#E   /c3?H&.CMM#(( #++""*&)$i#dnnT:6V2W&W#""%77;c30C0CM)6COOCHH %55""&BB!= 0.95) receive a confidence boost proportional to coverage. When nearly all non-ASCII bytes form valid multi-byte pairs, the structural evidence is strong and should increase the candidate's ranking relative to single-byte alternatives whose bigram models may score higher on small samples. Note: boosted confidence values may exceed 1.0 and are used only for relative ranking among candidates. ``run_pipeline`` clamps all confidence values to [0.0, 1.0] before returning to callers. c38K|]\}}|vs |ywNr)rr_sc enc_lookups rrz/_score_structural_candidates..s&*;YT3tz?Q 4*;s  c3:K|]}|jr|ywr)r)res rrz/_score_structural_candidates..sJ#3a1>>#3sNgffffff?c|jSr)r!xs rz._score_structural_candidates..sq||rTkeyreverse)rrrlistr_STAT_SCORE_MAX_BYTESr rrrr r!r"r#sort) rrrrrvalid_mb single_byteresultsboostedrcoveragers @r_score_structural_candidatesrs0,++aq~~ ++J*;HJ#3JJK4457P7PK7PQG &(G ;<::3??&&qzz373 t  NNJJ H =qzz1;;  NN1  LL+TL: N1+s D&D&rct|dkDr|djt|dj|r|dj}|dj}|ddD]}|jtvst |j||j |j}|Dcgc]}|j|k7s||us|}}|Dcgc]}|j|k(s|}}|g||cS|Scc}wcc}w)aDemote niche Latin encodings when no distinguishing bytes are present. Some bigram models (e.g. iso-8859-10, iso-8859-14, windows-1254) can win on data that contains only bytes shared with common Western Latin encodings. When there is no byte-level evidence for the winning encoding, promote the first common Western Latin candidate to the top and push the demoted encoding to last. rrN)rr rr!r'r r"r#) rrdemoted_encodingtop_confrpromotedrothersdemoted_entriess r_demote_niche_latinrs G q AJ   + 71:.. 5"1:..1:((Azz44*JJ!**akk '&!!**8H*HQVWZAw/6"XgGW9W1g"X <6.s"Q$6DAq!**:Pa$6s!!Nc38K|]}|dkDs |tvywr)rz)rrs rrz!_promote_koi8t..s A1D1& &s  )r next enumeraterr!r r"r#) rr koi8t_idx koi8t_resultrrrrrs r_promote_koi8trs gaj))X5QIg$6QSWXI A AAy) 1:(("  ! !   ! !  " "   !*' 2E 21a9n! 2E"6"" NFs  B3%B3ic|dk(r|S |j|djddS#tttf$rYywxYw)aPDecode data from encoding and re-encode as UTF-8 for language scoring. Returns None if the encoding is unknown. For UTF-8, returns data as-is. Uses ``errors="ignore"`` because the data already passed byte-validity filtering for the detected encoding; any residual invalid bytes are irrelevant for language scoring. utf-8ignorer surrogatepassN)rencoder TypeError ValueError)rr s r_to_utf8rs[7 {{8H{5<< O=   J /s#-AAc g}d}d}|D]G}|j}||jt|j}|?|r=t|jr(| t |}t ||j|\}}|R|rPtdrEt ||j}|r-||jdk7r t |}t |d|\}}|j} | |jdnd} ||jk7s| |jk7r3|jt|j|j|| 7|j|J|S)aFill in language and mime_type for results missing them. **Language** (only for text results where ``encoding is not None``): Tier 1: single-language encodings via hardcoded map (instant). Tier 2: multi-language encodings via statistical bigram scoring (lazy). Tier 3: decode to UTF-8, score against UTF-8 language models (universal fallback). **MIME type**: text results default to ``"text/plain"``, binary results (``encoding is None``) default to ``"application/octet-stream"``. N)profilerz text/plainr) r"r rrrrrr#rr r!) rrfilledr utf8_profileresultlang_ utf8_datamimes r_fill_metadatarsM%'F$(G)-L .>&> MM1B1BD$O  MM& !CD MrcJt||}t||}t||S)zGApply confusion resolution, niche Latin demotion, and KOI8-T promotion.)rrr)rrs r_postprocess_resultsrOs) 'tW5G!$0G $ ((rr&rinclude_encodingsexclude_encodingsno_match_encodingempty_input_encoding encoding_era max_bytesrrrrct}|d|}t|||}td|D} |s t|| dSt |} | | j | vr| gSt |} | | j | vr| gSt|} | | j | j | vr| gSt|} | | gSt|}t|}||t||rtgSt|}||j | vrt||| }|gS||j | vr|gS||j | vr|gSt||}|s t|| dSt!|||}|s t|| dSg}|D]f}|j"s|j$j'|j(}| t+|||}|dkDsJ|j-|j(|fh|rA|j/dd |d \}}|t0k\rt3||||}|r t5||S|dt6}t9t;|t=|}|s t|| dSt5||S) zBCore pipeline logic. Returns list of results sorted by confidence.Nc34K|]}|jywr)r)rrs rrz%_run_pipeline_core..ks'GJSJsr)rrrc |dS)Nrrrs rrz$_run_pipeline_core..sQqTrTrr)r r frozensetrrr rrrrrr_BINARY_RESULTrrrrrrrrrrr _STRUCTURAL_CONFIDENCE_THRESHOLDrrrrrr)rrrrrrrr candidatesr bom_resultutf1632_result escape_result magic_result utf8_precheckascii_precheckrrrrscorer best_scorer stat_datas r_run_pipeline_corerYs  C   D  .?ARSJ''GJ'GGG % '+A  D!J*"5"5"@| -T2N!n&=&=&H +40M!  " " .  " "g - %L~ %M"$'N   " di 0 *$/M ]%;%;w%F4T='R !n&=&=&H ]%;%;w%F*$ ; %&7BUVV,D2BCH %&7BUVV24   MM%%chh/E}0sC@s{!((#((E):;  >4@)!, : 9 92')93G+D':: ++,I#Iu5E/FGHG %&7BUVV g ..rc 2t|||||||}t|dt|}|s d}t||D cgc]S} | jdkDr@t | j t| jd| j| jn| Uc} Scc} w)aURun the full detection pipeline. :param data: The raw byte data to analyze. :param encoding_era: Filter candidates to a specific era of encodings. :param max_bytes: Maximum number of bytes to process. :param include_encodings: If not ``None``, only return these encodings. :param exclude_encodings: If not ``None``, never return these encodings. :param no_match_encoding: Encoding returned when no candidate survives. :param empty_input_encoding: Encoding returned for empty input. :returns: A list of :class:`DetectionResult` sorted by confidence descending. rNz/pipeline must always return at least one resultg?) rr_LANG_SCORE_MAX_BYTES RuntimeErrorr!r r minr"r#) rrrrrrrrmsgrs r run_pipeliners*! +++1GT"8#897CG ?3  A <<#   C c$:AJJ T     sAB)V__doc__rchardet._utilsr chardet.enumsrchardet.modelsrrrrchardet.pipeliner r r r r chardet.pipeline.asciirchardet.pipeline.binaryrchardet.pipeline.bomrchardet.pipeline.confusionrchardet.pipeline.escaperchardet.pipeline.magicrchardet.pipeline.markuprchardet.pipeline.statisticalrchardet.pipeline.structuralrrrchardet.pipeline.utf8rchardet.pipeline.utf1632rchardet.pipeline.validityrchardet.registryrrrr r rr r'str__annotations__rVintr]r`rorqdictrzrbytesrrrboolrrrrrrrrfloatrrrrrrrrrrrrr7s,% 0-+?:/99 .<8CC ' ( $( +4+3/8/1/Ys^1n/8 "/Ys^"T09(0in-6-9S>:.-** 3d3 #./*3L* #/T#s(^ ! !"!s^! !HPP s^PP / P* BS B B$ B$   "3 3L#-.3 3 <  3l/ /E#u*-./L#-./  /  / /d  / " /@  / " /H5CEDL$3 3/3 /3l) ) / ") /)'J/ 04/3% 'J/ J/J/J/ !~, J/ !~, J/J/J/ /J/`', 04/3% ', ,,, !~, , !~, ,,, /,r