U
|
M ±dI ã @ sZ d dl mZ d dlmZ d dlmZmZ ddlmZm Z m
|
Z
|
ddl m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZ G dd dZ G d d
|
d
|
e Z G d d d e Z G d d de Z G dd de Z G dd de Z!G dd de Z"G dd de Z#G dd de Z$eddee% ee% e&dd d Z'ed dd%e%e(e&e(d!d"d#Z)d$S )&é )Ú lru_cache)Ú getLogger)ÚListÚOptionalé )Ú COMMON_SAFE_ASCII_CHARACTERSÚTRACEÚ UNICODE_SECONDARY_RANGE_KEYWORD)Úis_accentuatedÚis_asciiÚis_case_variableÚis_cjkÚ is_emoticonÚ is_hangulÚ is_hiraganaÚ is_katakanaÚis_latinÚis_punctuationÚ is_separatorÚ is_symbolÚis_thaiÚis_unprintableÚ remove_accentÚ unicode_rangec @ sP e Zd ZdZeedddZeddddZddd d
|
Ze e
|
dd d Z dS ) ÚMessDetectorPluginzy
|
Base abstract class used for mess detection plugins.
|
All detectors MUST extend and implement given methods.
|
©Ú characterÚreturnc C s t dS )z@
|
Determine if given character should be fed in.
|
N©ÚNotImplementedError©Úselfr © r" úLd:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\charset_normalizer/md.pyÚeligible$ s zMessDetectorPlugin.eligibleNc C s t dS )z
|
The main routine to be executed upon character.
|
Insert the logic in witch the text would be considered chaotic.
|
Nr r r" r" r# Úfeed* s zMessDetectorPlugin.feed©r c C s t dS )zB
|
Permit to reset the plugin to the initial state.
|
Nr ©r! r" r" r# Úreset1 s zMessDetectorPlugin.resetc C s t dS )z
|
Compute the chaos ratio based on what your feed() has seen.
|
Must NOT be lower than 0.; No restriction gt 0.
|
Nr r' r" r" r# Úratio7 s zMessDetectorPlugin.ratio) Ú__name__Ú
|
__module__Ú __qualname__Ú__doc__ÚstrÚboolr$ r% r( ÚpropertyÚfloatr) r" r" r" r# r s r c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )Ú TooManySymbolOrPunctuationPluginNr& c C s" d| _ d| _d| _d | _d| _d S )Nr F)Ú_punctuation_countÚ _symbol_countÚ_character_countÚ_last_printable_charZ_frenzy_symbol_in_wordr' r" r" r# Ú__init__A s
|
z)TooManySymbolOrPunctuationPlugin.__init__r c C s | ¡ S ©N©Ú isprintabler r" r" r# r$ I s z)TooManySymbolOrPunctuationPlugin.eligiblec C sp | j d7 _ || jkrf|tkrft|r8| jd7 _n.| ¡ dkrft|rft|dkrf| jd7 _|| _d S )Nr Fé ) r5 r6 r r r3 Úisdigitr r r4 r r" r" r# r% L s ÿþ
|
ÿþ
|
ýz%TooManySymbolOrPunctuationPlugin.feedc C s d| _ d| _d| _d S ©Nr )r3 r5 r4 r' r" r" r# r( ^ s z&TooManySymbolOrPunctuationPlugin.resetc C s0 | j dkrdS | j| j | j }|dkr,|S dS )Nr ç g333333Ó?)r5 r3 r4 )r! Zratio_of_punctuationr" r" r# r) c s
|
|
þz&TooManySymbolOrPunctuationPlugin.ratio© r* r+ r, r7 r. r/ r$ r% r( r0 r1 r) r" r" r" r# r2 @ s r2 c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )ÚTooManyAccentuatedPluginNr& c C s d| _ d| _d S r= ©r5 Ú_accentuated_countr' r" r" r# r7 p s z!TooManyAccentuatedPlugin.__init__r c C s | ¡ S r8 )Úisalphar r" r" r# r$ t s z!TooManyAccentuatedPlugin.eligiblec C s( | j d7 _ t|r$| jd7 _d S ©Nr )r5 r
|
rB r r" r" r# r% w s z TooManyAccentuatedPlugin.feedc C s d| _ d| _d S r= rA r' r" r" r# r( } s z TooManyAccentuatedPlugin.resetc C s4 | j dks| j dk rdS | j| j }|dkr0|S dS )Nr é r> gffffffÖ?rA )r! Zratio_of_accentuationr" r" r# r) s z TooManyAccentuatedPlugin.ratior? r" r" r" r# r@ o s r@ c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )ÚUnprintablePluginNr& c C s d| _ d| _d S r= )Ú_unprintable_countr5 r' r" r" r# r7 s zUnprintablePlugin.__init__r c C s dS ©NTr" r r" r" r# r$ s zUnprintablePlugin.eligiblec C s( t |r| jd7 _| jd7 _d S rD )r rG r5 r r" r" r# r% s zUnprintablePlugin.feedc C s
|
d| _ d S r= )rG r' r" r" r# r( s zUnprintablePlugin.resetc C s | j dkrdS | jd | j S )Nr r> rE )r5 rG r' r" r" r# r) s
|
zUnprintablePlugin.ratior? r" r" r" r# rF s rF c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )Ú SuspiciousDuplicateAccentPluginNr& c C s d| _ d| _d | _d S r= ©Ú_successive_countr5 Ú_last_latin_characterr' r" r" r# r7 ¢ s z(SuspiciousDuplicateAccentPlugin.__init__r c C s | ¡ ot|S r8 )rC r r r" r" r# r$ ¨ s z(SuspiciousDuplicateAccentPlugin.eligiblec C st | j d7 _ | jd k rjt|rjt| jrj| ¡ rJ| j ¡ rJ| jd7 _t|t| jkrj| jd7 _|| _d S rD )r5 rL r
|
ÚisupperrK r r r" r" r# r% « s ÿþýz$SuspiciousDuplicateAccentPlugin.feedc C s d| _ d| _d | _d S r= rJ r' r" r" r# r( ¹ s z%SuspiciousDuplicateAccentPlugin.resetc C s | j dkrdS | jd | j S )Nr r> r; )r5 rK r' r" r" r# r) ¾ s
|
z%SuspiciousDuplicateAccentPlugin.ratior? r" r" r" r# rI ¡ s rI c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )ÚSuspiciousRangeNr& c C s d| _ d| _d | _d S r= )Ú"_suspicious_successive_range_countr5 Ú_last_printable_seenr' r" r" r# r7 Ç s zSuspiciousRange.__init__r c C s | ¡ S r8 r9 r r" r" r# r$ Ì s zSuspiciousRange.eligiblec C sx | j d7 _ | ¡ s&t|s&|tkr0d | _d S | jd krD|| _d S t| j}t|}t||rn| jd7 _|| _d S rD )r5 Úisspacer r rP r Ú is_suspiciously_successive_rangerO )r! r Úunicode_range_aÚunicode_range_br" r" r# r% Ï s" ÿþý
|
|
|
zSuspiciousRange.feedc C s d| _ d| _d | _d S r= )r5 rO rP r' r" r" r# r( æ s zSuspiciousRange.resetc C s. | j dkrdS | jd | j }|dk r*dS |S )Nr r> r; g¹?)r5 rO )r! Z ratio_of_suspicious_range_usager" r" r# r) ë s
|
þzSuspiciousRange.ratior? r" r" r" r# rN Æ s rN c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )ÚSuperWeirdWordPluginNr& c C s: d| _ d| _d| _d| _d| _d| _d| _d| _d| _d S )Nr FÚ ) Ú _word_countÚ_bad_word_countÚ_foreign_long_countÚ_is_current_word_badÚ_foreign_long_watchr5 Ú_bad_character_countÚ_bufferÚ_buffer_accent_countr' r" r" r# r7 û s z SuperWeirdWordPlugin.__init__r c C s dS rH r" r r" r" r# r$ s z SuperWeirdWordPlugin.eligiblec C sÚ | ¡ r| j|7 _t|r,| jd7 _| jdkrt|dksJt|rt|dkrt|dkrt|dkrt |dkrt
|
|dkrd| _d S | jsd S | ¡ s´t |s´t |r | jr | jd7 _t| j}| j|7 _|dkr6| j| dkrd| _t| jd r6| jd ¡ r6| jd7 _d| _|dkr\| jr\| jd7 _d| _| jr| jd7 _| jt| j7 _d| _d| _d| _d | _n6|d
|
krÖ| ¡ dkrÖt|rÖd| _| j|7 _d S ) Nr FTé gÃõ(\ÂÕ?éÿÿÿÿé rV r > ú|ú<ú>ú-ú~Ú_ú=)rC r] r
|
r^ r[ r r r r r r rQ r r rW Úlenr5 rZ rM rY rX r\ r< r )r! r Z buffer_lengthr" r" r# r% sv ÿ
|
þþ
|
ý
|
ü
|
û
|
ú
|
ù ÿÿÿþ
|
|
ÿ
|
þýzSuperWeirdWordPlugin.feedc C s4 d| _ d| _d| _d| _d| _d| _d| _d| _d S )NrV Fr )r] rZ r[ rX rW r5 r\ rY r' r" r" r# r( B s zSuperWeirdWordPlugin.resetc C s$ | j dkr| jdkrdS | j| j S )Né
|
r r> )rW rY r\ r5 r' r" r" r# r) L s zSuperWeirdWordPlugin.ratior? r" r" r" r# rU ú s 6
|
rU c @ s^ e Zd ZdZddddZeedddZeddd d
|
Zddd d Z e
|
e dd dZ dS )ÚCjkInvalidStopPluginu²
|
GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
|
can be easily detected. Searching for the overuse of 'ä¸
' and 'ä¸'.
|
Nr& c C s d| _ d| _d S r= ©Ú_wrong_stop_countÚ_cjk_character_countr' r" r" r# r7 Z s z CjkInvalidStopPlugin.__init__r c C s dS rH r" r r" r" r# r$ ^ s z CjkInvalidStopPlugin.eligiblec C s4 |dkr| j d7 _ d S t|r0| jd7 _d S )N> õ ä¸õ ä¸
r )rm r rn r r" r" r# r% a s
|
zCjkInvalidStopPlugin.feedc C s d| _ d| _d S r= rl r' r" r" r# r( h s zCjkInvalidStopPlugin.resetc C s | j dk rdS | j| j S )Né r> )rn rm r' r" r" r# r) l s
|
zCjkInvalidStopPlugin.ratio) r* r+ r, r- r7 r. r/ r$ r% r( r0 r1 r) r" r" r" r# rk T s rk c @ sZ e Zd ZddddZeedddZedddd Zddd
|
d Ze e
|
dd d Z dS )ÚArchaicUpperLowerPluginNr& c C s. d| _ d| _d| _d| _d| _d | _d| _d S )NFr T)Ú_bufÚ _character_count_since_last_sepÚ _successive_upper_lower_countÚ#_successive_upper_lower_count_finalr5 Ú_last_alpha_seenÚ_current_ascii_onlyr' r" r" r# r7 t s z ArchaicUpperLowerPlugin.__init__r c C s dS rH r" r r" r" r# r$ s z ArchaicUpperLowerPlugin.eligiblec C s$ | ¡ ot|}|dk}|r| jdkr| jdkrV| ¡ dkrV| jdkrV| j| j7 _d| _d| _d | _d| _| j d7 _ d| _d S | jdkr¢t
|
|dkr¢d| _| jd k rþ| ¡ r¾| j ¡ sÐ| ¡ rø| j ¡ rø| jdkrð| jd7 _d| _qþd| _nd| _| j d7 _ | jd7 _|| _d S )NFr é@ r Tr; ) rC r rt r< rx rv ru rw rs r5 r rM Úislower)r! r Z is_concernedZ chunk_sepr" r" r# r% sF ÿ
|
þýÿ
|
ÿÿ
|
z ArchaicUpperLowerPlugin.feedc C s. d| _ d| _d| _d| _d | _d| _d| _d S )Nr FT)r5 rt ru rv rw rs rx r' r" r" r# r( ® s z ArchaicUpperLowerPlugin.resetc C s | j dkrdS | j| j S )Nr r> )r5 rv r' r" r" r# r) · s
|
z ArchaicUpperLowerPlugin.ratior? r" r" r" r# rr s s * rr é )Úmaxsize)rS rT r c C s| | dks|dkrdS | |kr dS d| kr4d|kr4dS d| ksDd|krHdS d| ksXd|krld| kshd|krldS | d¡| d¡ }}|D ] }|tkrq||kr dS q| dk|dk }}|s¾|rÒd | ksÎd |krÒdS |rÞ|rÞdS d
|
| ksðd
|
|kr d | ksd |krdS | d ks |d kr dS d | ksHd |ksH| d krx|d krxd | ks\d |kr`dS d| kstd|krxdS dS )za
|
Determine if two Unicode range seen next to each other can be considered as suspicious.
|
NTFZLatinZ EmoticonsZ Combiningú )ÚHiraganaÚKatakanaÚCJKZHangulz Basic Latin)r r~ Z PunctuationZForms)Úsplitr )rS rT Zkeywords_range_aZkeywords_range_bÚelZrange_a_jp_charsZrange_b_jp_charsr" r" r# rR ¿ sh ÿÿÿþÿúÿÿÿþrR i çÉ?F)Údecoded_sequenceÚmaximum_thresholdÚdebugr c C sX dd t ¡ D }t| d }d}|dk r0d}n|dkr>d}nd }t| d
|
t|D ]d\}}|D ]} | |¡r`| |¡ q`|d kr|| d ks||d krTtd d |D }||krT qºqT|rNtd}
|
|
|
t
|
d| d| d| ¡ t| dkr(|
|
t
|
d| dd
¡ |
|
t
|
d| dd
¡ |D ] } |
|
t
|
| j d| j ¡ q,t |dS )zw
|
Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
|
c S s g | ]
|
}| qS r" r" )Ú.0Zmd_classr" r" r# Ú
|
<listcomp> s z mess_ratio.<locals>.<listcomp>r r> i é r{ ry é Ú
|
r c s s | ] }|j V qd S r8 )r) )r Údtr" r" r# Ú <genexpr>% s z mess_ratio.<locals>.<genexpr>Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=rq zStarting with: Nz Ending with: iðÿÿÿz: é )r Ú__subclasses__ri ÚzipÚranger$ r% Úsumr Úlogr Ú __class__r) Úround) r r
r Z detectorsÚlengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr ÚindexÚdetectorÚloggerr r" r" r# Ú
|
mess_ratio sF ÿ
|
ÿ
|
ÿ
|
þþ r N)r F)*Ú functoolsr Úloggingr Útypingr r Zconstantr r r Úutilsr
|
r r r r r r r r r r r r r r r r r2 r@ rF rI rN rU rk rr r. r/ rR r1 r r" r" r" r# Ú<module> s6 H"/%4Z L þF ÿ þ
|
