^j~UdZddlZddlZddlZddlZddlZddlmZm Z m Z m Z m Z ddl mZddlmZeeeefeeeeeeeeffffZiddddd d d d d ddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4 Zeeefed5<ej3Dcic]\}}|| c}}Zeeefed6<d7ed8efd9Zej:d8efd:Ziddddd dd dddddddddddddd dd"dd$dd&dd'dd)dd(dd d d d d d dddddddd; Zeeefed<<d Z dZ!d Z"d=Z#d Z$ej:d>ed8efd?Z%d@edAedBedCed8ef dDZ&d7edEedFedGeedHeeeeeffd8eedzeeeff dIZ'efd7edJedKedLeed8edzf dMZ(d7edEedFedGeedHeeeeeffd8edzf dNZ)ej:dGeed8eeeffdOZ*eZ+ej:dPed8eefdQZ,dEedFedLeed8eedzfdRZ-dSe dTeee.eedzeeffdPedLeedzd8e/f dUZ0d7edGeed8e dzfdVZ1efd7edEedFedGeedLeed8edzf dWZ2ej:dEedFed8eefdXZ3ej:dEedFedGeed8eeeeefffdYZ4d7edEedFedGeedZeedzd[eedzd8edzfd\Z5d]edEedFed8eeefdzfd^Z6d_Z7d`Z8daZ9dbZ:d7edce.ed8e.efddZ;ycc}}w)eaConfusion group resolution for similar single-byte encodings. At runtime, loads pre-computed distinguishing byte maps from confusion.bin and uses them to resolve statistical scoring ties between similar encodings. Build-time computation (``compute_confusion_groups``, ``compute_distinguishing_maps``, ``serialize_confusion_data``) lives in ``scripts/confusion_training.py``. Note: ``from __future__ import annotations`` is intentionally omitted because this module is compiled with mypyc, which does not support PEP 563 string annotations. N) ART_LANGUAGE BigramProfile get_enc_indexget_idf_weightsscore_with_profile)DetectionResult)lookup_encodingLuLlLtLmLoMnMcMeNd Nl No Pc Pd PsPePiPfPoSmScSkSoZsZlZpCcCfCsCoCn) _INT_TO_CATEGORYCATEGORY_TO_INTdatareturnci}d}tjd||\}|dz }t|D]}tjd||\}|dz }||||zjd}||z }tjd||\}|dz }||||zjd}||z }tjd||\} |dz }g} i} t| D]b}tjd||\} } }|dz }| j | t j | d t j |d f| | <dt| | f|||f<|S) zLoad confusion group data from raw bytes. :param data: The raw binary content of a confusion.bin file. :returns: A :data:`DistinguishingMaps` dictionary keyed by encoding pairs. rz!Hr z!Br zutf-8z!BBBrr7)struct unpack_fromrangedecodeappendrEget frozenset)rGresultoffset num_pairs_ name_a_lenname_a name_b_lenname_b num_diffsdiff_bytes_list categoriesbv cat_a_int cat_b_ints \/var/www/html/metafoliadorv2/venv/lib/python3.12/site-packages/chardet/pipeline/confusion.py%deserialize_confusion_data_from_bytesr`Ns "$F F%%dD&9LY aKF 9 L**4v> ! fv 23::7C***4v> ! fv 23::7C*))$f= ! %'13 y! A'-'9'9&$'O $B 9 aKF  " "2 & $$Y5 $$Y5JrN  %.o$> #K 1L4 Mctjjdjd}|j }|st j dtdiS t|}i}|jD]-\\}}}t|xs|} t|xs|} ||| | f</|S#tjtf$r}d|}t||d}~wwxYw)zLoad confusion group data from the bundled confusion.bin file. :returns: A :data:`DistinguishingMaps` dictionary keyed by encoding pairs. zchardet.modelsz confusion.binuZchardet confusion.bin is empty — confusion resolution disabled; reinstall chardet to fixr ) stacklevelzcorrupt confusion.bin: N) importlib resourcesfilesjoinpath read_byteswarningswarnRuntimeWarningr`rJerrorUnicodeDecodeError ValueErroritemsr ) refrawraw_mapsemsg normalizedabvaluenorm_anorm_bs r_load_confusion_datar{vs    # #$4 5 > > OC .. C   '    %8= &(J!)- A #(q #(q', FF#$-  LL, -%'s+o1$%s B//CCC) r.r/r0r1r2r4r3r6r5r7rrr_CATEGORY_PREFERENCEencodingc&td}tdD]_} t|gj|}t |dk7r.t j|}|dk(rd||<N|ddk(s|dvs[d||<at|S#t$rYxwxYw)u256-entry table: 0 = non-letter, 1 = uppercase letter, 2 = other letter. Combining marks count as letters: in decomposed text (Vietnamese under windows-1258) a base letter's neighbor is its diacritic, which is word-internal, not a word boundary. Whitespace deliberately counts as a plain non-letter: exempting space-adjacent letters from the isolated-letter demotion (to spare one-letter words like French ``à``) was tried and falsified by the accuracy suite — Irish/Finnish po files and the EBCDIC record set depend on space-adjacent demotions, so the cross-family decisive-override gate handles the ``à`` failure mode instead. r}r r rL)rrr ) bytearrayrLbytesrMrmlen unicodedatacategory)r~tablerwchcats r__letter_case_tablers cNE 3Z s""8,B r7a< ""2& $;E!H Vs]c\1E!H <"   sB BBrleftright case_tablectj|d}|ddk7r|S||}||}|dk(r |dk(rtS|dk(r |dk(rtS|S)aPreference for reading a byte as *cat*, adjusted for word shape. A letter reading only deserves its high preference when its neighbors make it look like part of a word under the same encoding: a letter with no letter neighbors is quoted/isolated punctuation in disguise, and a lowercase letter immediately followed by an uppercase one is not a word shape any of the supported languages produce. rrr r )r|rO_IMPLAUSIBLE_LETTER_PREFERENCE)rrrrpref left_kind right_kinds r__context_preferencerse # #C +D 1v} 4 IE"JA~*/-- d{zQ-- Kraenc_aenc_b diff_bytesr[c0t|\}}t|jd|}|syt|}t|} d} d} d} d} d}d}t |dz }|D]}||\}}t |f}|j |}d}|dk\s1|tks;|dkDr||dz nd}||kr||dznd}t||||}t|||| }||kDr8|ddk(r |ddk(rnc| ||z z } |ddk(rS|tk(rJ| ||z z } |dz }n<||kDr7|ddk(r |ddk(rn&| ||z z } |ddk(r|tk(r | ||z z } |dz }|dz }|j ||dz}|dk\s|tkrސ| | kDr || | z | |fS| | kDr || | z | |fSy)uSContext-aware category voting. Returns ``(winner, margin, demotion_margin, demotion_events)``. For each occurrence of a distinguishing byte, compare the two encodings' readings: Unicode category preference, adjusted for word shape (see :func:`_context_preference`). The reading that makes more linguistic sense of the byte *in its context* collects the vote; occurrences vote independently, so repeated evidence counts. ``demotion_margin`` counts only the winner's votes earned where the *losing* side's letter reading was word-shape-implausible — evidence against an impossible reading, which is far stronger than the naive letters-beat-symbols preference. ``demotion_events`` counts how many distinct occurrences contributed to it, so callers can tell repeated evidence from one loud byte. N)Nrrrrr rP) _pair_byte_tablesrP translaterrrfind_MAX_VOTE_OCCURRENCESrr)rGrrrr[non_diffrTrelevanttable_atable_bvotes_avotes_b demotion_a demotion_bevents_aevents_bendr\cat_acat_bneedleposexaminedrrpref_apref_bs r__vote_with_marginr s8$J/KHah78H  'G 'GGGJJHH d)a-C"-!"~ uuiiQh8&;;$'!G4a=D%(3YDqMAE(eWEF(eWEF8s?uQx3v.GQx365S+S"fvo5  A &8s?uQx3v.GQx365S+S"fvo5  A  MH))FC!G,C;Qh8&;; "-Fg'X==g'X== raenc_xenc_y languagesc*t}t|||}|y||\}}|\}} t||| ||\} } } } | &| tk\r| tk\rt |t kr| St||| ||}t |t k\r ||| k(r|Sy||S| S)uReturn the byte-evidence winner between two encodings, or ``None``. Mirrors the in-band pairwise rule of :func:`resolve_confusion_groups` (decisive demotion vote, else bigram rescore, else category vote) for callers outside the ranked-results scan — e.g. the classic-Mac line-ending promotion, whose platform prior must not override distinguishing-byte evidence. Returns ``None`` when the pair has no distinguishing map or the evidence is inconclusive. *languages* must carry what the two results being compared report, or the mirror breaks: the rescore would arbitrate the same pair under a different rule than the confusion stage just did, and a veto built on that answer can reverse a promotion the stage had settled. N)r{_find_pair_keyr_DECISIVE_VOTE_MARGIN_DECISIVE_MIN_EVENTSr_CROSS_FAMILY_MIN_DIFFSresolve_by_bigram_rescore)rGrrrmapspair_keyrr[rr cat_winner_margindemotion_margindemotion_events bigram_winners r_confusion_pair_winnerr`s(  DdE51H!(^J LE5z$_pair_byte_tables..s<1&)Q.s q!t?Os )rPrrO)rs r__modelled_languagesrs- ,223; rac|syt|}t|}||z}||ksyt|vst|vr |thzS|S)uLanguages to score *enc_a* and *enc_b* under, or ``None`` for all. ``None`` means *unrestricted* — score every variant, the original max-over-models comparison. It is not an abstention. An encoding should not win on language coverage the other side lacks. On a Hungarian document the u-double-acute bytes score 0.014 against iso8859-2's *Czech* model (which reads them as a common r-hacek) and only 0.006 against iso8859-16's Hungarian one, so max-over-variants hands Hungarian text to the Czech reading. Dropping the languages only one side models removes that particular unfairness. This is deliberately a narrow rule, and two tempting generalisations were measured and rejected against the accuracy suite: * Restricting to *languages* themselves rather than the shared set costs 12 tests. As a consequence the restriction is a no-op for pairs whose language coverage already matches (67 of the 236 confusion pairs), which is accepted — those pairs have no coverage asymmetry to correct in the first place. * Restricting when *languages* is not wholly inside the shared set costs 6 tests: cp1125 models only Ukrainian, so a Belarusian cp866 document shares just ``uk`` with it, and scoring the *right* encoding under the *wrong* language loses to cp1125. Falling back to unrestricted is also what keeps a Vietnamese windows-1258 document, whose rival cp1252 models no Vietnamese, resolving correctly. The cost is that pairs modelling disjoint languages (koi8-r/koi8-u, mac-roman/mac-turkish) never restrict at all — about 20% of calls over the corpus. N)r _ART_LANGUAGE)rrrlangs_alangs_bshareds r__comparable_languagesrsWF !%(G!%(G w F  =G#;'' Mraprofileindexc\|j|}|sytfd|DdS)aReturn the best bigram score for *enc*, restricted to *languages*. *languages* of ``None`` means every variant, the unrestricted max-over-models comparison. The ``default`` guards a caller-supplied set that names no variant of *enc*; note that scoring 0.0 hands the comparison to the rival rather than abstaining, so callers wanting an abstention must not rely on it. gc3LK|]\}}}|vrt||ywr)r)rrmodel model_keyrrs r_rz&_best_variant_score..s5 &eY DI$5 wy 9 s!$)default)rOmax)rrrrvariantss` ` r__best_variant_scorers7yy~H   *2   rac.t|dkryt|\}}t|jd|}|syt}i}t|dz }|tzt|krt }t |jd|D]l} t| f} |j| } | dk\s&| r|j| dz | |kr|j| |j| | dz} | dk\rGn|D].} || dz|| dzz} |j| d|| z|| <0nLt|D]>} || }|| dz}||||zs|dz|z} |j| d|| z|| <@|sytj|S)aBuild the bigram profile of *data* restricted to *diff_bytes* context. The profile holds only bigrams where at least one byte is in *diff_bytes*, weighted by IDF like the full-input profile, so scoring a model against it asks how well that model explains the bytes two encodings read differently, and nothing else. Returns ``None`` when *data* cannot form a bigram or contains no distinguishing byte. :param data: The raw byte data to examine. :param diff_bytes: Byte values where the two encodings differ. r Nr rr)rrrr_DENSE_HIT_DIVISORsetrPrraddrOrLrfrom_weighted_freq)rGrris_diffhitsidffreqlimitstartsr\rriidxb1b2s r_build_focused_profilers 4y1} **5Hg t~~dH- .D   CD IME   3t9, 5DNN4:; 1BB5\F))F#C(JJsQw';JJsOiia0 ( 1 4A7a<4A;.Ca(3s83DI 4u 4AaBa!eBBK'"+-7b.Ca(3s83DI  4   + +D 11ract||}|yt|||}t}t||||}t||||} || kDr|S| |kDr|Sy)uResolve between two encodings by re-scoring only distinguishing bigrams. Builds a focused bigram profile containing only bigrams where at least one byte is a distinguishing byte, then scores both encodings under the languages they can be compared in (see :func:`_comparable_languages`). There is no abstention path here: when the pair has no comparable language the comparison widens to every variant rather than declining to answer. That is why a Danish mac-roman/mac-turkish document, whose pair models disjoint languages, is still decided by the Turkish model the rescore has no business consulting — the caller's category vote is better placed on such evidence, and overriding this to abstain was measured as costing more accuracy than it recovers. :param data: The raw byte data to examine. :param enc_a: First encoding name. :param enc_b: Second encoding name. :param diff_bytes: Byte values where the two encodings differ. :param languages: Languages the ranked results report for this pair; empty when the caller has no ranking to draw on, which scores every variant. :returns: The winning encoding name, or ``None`` if tied or if no distinguishing byte occurs in *data*. N)rrrr) rGrrrrr comparablerbest_abest_bs r_rrRsh>$D*5G&ueY?J OE % CF % CF    rac t}tddD]I}t|f} |j|} |j|}||k7s9|j |Kt |S#t$rd}YCwxYw#t$rd}YBwxYw)aByte values >= 0x80 that *enc_a* and *enc_b* decode to different text. The distinguishing set for a pair the confusion maps do not cover, computed from the codecs themselves. A byte only one side can decode counts as differing. Bytes below 0x80 are left out: the callers ask about high-byte evidence, and every single-byte Latin family agrees on ASCII anyway. r}N)rrLrrMrmrrP)rroutrwrqtext_atext_bs r_differing_high_bytesrsEC 4  QDk !$E!2F !$E!2F V  GGAJ  S>" F " F s"A.A?. A<;A<? B  B c(i}|D]j}g}||fD]R} t|gj|}|jt |dk(rt j |ndT|d|df||<l|S#t$r|jdYwxYw)aUnicode general categories of each distinguishing byte under both encodings. The confusion maps carry this table for their own pairs; callers that arbitrate a pair the maps do not cover (the niche-Latin demotion's candidate against its swap target) build it here. A byte one side cannot decode reads as unassigned (``Cn``), which the vote treats as the least plausible reading of all. r7r r)rrMrmrNrrr)rrrrrwcatsrrs r__pair_categoriesrs)+E  &5> LC A3Z&&s+ KKCGqL ,,R0d K  LGT!W%a & L &  D! sA44BB languages_a languages_bc t||}|4t}t||||}t||||} || kDr|S| |kDr|St||||t |||\} } } } | S)aDecide a pair on its distinguishing bytes: model rescore, then context. The in-band rule of :func:`resolve_confusion_groups` for a pair the confusion maps do not cover, with one difference: each side is scored under the languages the caller names for it rather than the shared set. The bigram rescore decides when the models have an opinion; when they score the distinguishing bigrams equally (usually both at zero, a byte neither model has seen in that context), the category vote reads word shape instead, so a letter between letters still beats a superscript between letters. Returns ``None`` when neither step can tell the two apart. :param data: The raw byte data to examine. :param enc_a: First encoding name. :param enc_b: Second encoding name. :param diff_bytes: Byte values where the two encodings differ. :param languages_a: Variants of *enc_a* to score; ``None`` for all. :param languages_b: Variants of *enc_b* to score; ``None`` for all. )rrrrr)rGrrrrrrrrrrrrrs r_arbitrate_distinguishing_bytesrs8$D*5G$WeUKH$WeUKH F?L F?L*; eUJ(8z(R+'FGY Mrarc,||f|vr||fS||f|vr||fSy)zEFind the canonical key for a pair of encodings in the confusion maps.Nr)rrrs r_rrs3 u~u~ u~u~ ra4g{Gzt?g?g?resultsct|dkr|S|d}|j|S|jtk(r|St }|j }|t z}d}|}|j}tdt|D]} || } | j| dk(xs|| j z tk} | s|tk\s| j |krn@t||| j} | j|| \} }| \}}t|||| |\}}}}|'|tk\r|tk\rt| tkr|}n^td|j| jfD}t!|||| |}| rt| tkr||n|}n |||k(r|}nd}||| jk7r%| rVt#| j|| j| j$}t'|Dcgc] \}}|| k7s |}}}|g|cS| }| }| j}|dk(r|St#|j||j|j$}t'|Dcgc] \}}||k7s |}}}|g|Scc}}wcc}}w)a Resolve confusion between similar encodings in the top results. Checks the top result against each candidate within a confidence band. Always checks position 1 (preserving original top-2 behavior); for positions 2+ only checks within the band. Uses bigram re-scoring with category voting as fallback. :param data: The raw byte data to examine. :param results: Detection results sorted by confidence descending. :returns: A reordered list of :class:`DetectionResult` with the winner first. r rNr c3$K|]}|| ywrr)rrs r_rz+resolve_confusion_groups..qs#s)rr~languagerr{ confidenceCONFUSION_FLOOR_RATIOrLCONFUSION_BANDSTRICT_TIER_MAX_CONFrrrrrrPrr mime_type enumerate)rGrtoprtop_conffloor champion_idxchampion champion_encr candidatein_bandrrr[rrr _vote_marginrrrlangsrpromotedjrrests r_resolve_confusion_groupsrs 7|a !*C || ||}$  D~~H , ,ELH <Vy'9'99  '""""## H #,G"4?$!QQA?D?$t$ $  )) y\*|q H $G, B$!Q\0AA BD B  t 5@2 Cs I(,I( I.I.)<__doc__ functoolsimportlib.resourcesrdrJrrichardet.modelsrrrrrchardet.pipelinerchardet.registryr dicttuplestrrPintDistinguishingMapsrE__annotations__rorFrr`cacher{r|rrrrrrrrrrrrrrlistfloatrrrrrrrrr r r r)kvs00r_r.sO  -, #s(O )C.$sE#s(O34 457$t$t$t$t $ t $ t $t$t$t$t$$$$$$ !$"#$$             =$$sCx.H5E4J4J4L"MDAq1a4"Mc3hM%%:L%P />("("( "( ! (  ! (  ! ( !( !( !( !( !( !( !( !( !(  !!(" !#($ =(d38nH"# @SCUs*P P P P# P S%S/)* P  3:sC $% Pn!* +F +F +F +F~ +F  4Z +F\   #  S%S/)*   4Z "  #)C. #U5%<5H # #&  SYs^2 2 2~2s^d 2j  T%d E3 6788 9 ~$   6A2 A2&s^A2TA2R!* - - - -# - ~ -  4Z -` CIcN2  (1# #uS#X 2( ( ( (# ( 3$& (3$&( 4Z(V        38_t (W W / "W /WO#Ns2 L