U
|
ý°dg' ã @ s* d Z ddlmZmZ ddlZddlZddlZddlZddl Z ddl
|
m Z m Z ddl mZ e j d¡Ze j d¡Zdd Zed d
|
Zee j d d d g¡e j ddddg¡dd Zdd Zdd Ze j ddi e ddgifd d d ie ddgifd d dgie dd!d"gifd#dgd$d%e ddgifd#dgd&d%e dejdgifg¡d'd( Zd)d* Z e j d+d,d-d.g¡d/d0 Z ee j d1d$d&g¡d2d3 Z ed4d5 Z e j dd6dd7d8d9g¡d:d; Z ee j dd6ddd<d=g¡d>d? Z!ed@dA Z"edBdC Z#e j d¡e j dDdEdFg¡dGdH Z$dS )IzZ
|
Tests encoding functionality during parsing
|
for all of the parsers defined in parsers.py
|
é )ÚBytesIOÚ TextIOWrapperN)Ú DataFrameÚread_csvZ pyarrow_skipZ pyarrow_xfailc C sL d}| }t d |¡}|j|d|d}tddggddgd }t ||¡ d S )
|
NÚcp1255u ש×××:1234
|
562:123ú:)ÚsepÚencodingi2 é{ u ש×××Z1234©Úcolumns©r Úencoder r ÚtmÚassert_frame_equal)Ú all_parsersr ÚparserÚdataÚresultÚexpected© r ú[d:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\pandas/tests/io/parser/test_encoding.pyÚtest_bytes_io_input s r c C s@ | }t d ¡ }|j|ddd d}tddgg}t ||¡ d S )Nu Åaski, Jan;1ú;úutf-8)r r Úheaderu Åaski, Jané r )r r r r r r r r Útest_read_csv_unicode% s
|
r r ú,ú r úutf-16zutf-16lezutf-16bec
|
C sÌ | }d d|¡}dt ¡ d}|dd}d}t |¡}| |¡}t|d} | |¡ W 5 Q R X tt | |¡|d 2}
|
|j
|
|fd
|
|i|} |j
|
|
|
fd
|
|i|} W 5 Q R X t | | ¡ W 5 Q R X d S ) Nz)skip this
|
skip this too
|
A,B,C
|
1,2,3
|
4,5,6r Ú__z__.csvé )r Zskiprowsr Úwb©r r ) ÚreplaceÚuuidÚuuid4r Ú ensure_cleanr ÚopenÚwriter r r r ) r r r r r ÚpathÚkwargsÚutf8Ú
|
bytes_dataÚfZ bytes_bufferr r r r r Útest_utf16_bom_skiprows/ s û
|
|
r0 c C s6 t j |d¡}| }|j|ddd}t|dks2td S )Nz utf16_ex.txtr r )r r é2 )Úosr+ Újoinr ÚlenÚAssertionError)r Ú csv_dir_pathr+ r r r r r Útest_utf16_exampleL s r7 c C sL t j |d¡}| }|j|d dd}| d¡}|d d }d}||ksHtd S )Núunicode_series.csvúlatin-1)r r r r i` u$ à köldum klaka (Cold Fever) (1994))r2 r+ r3 r Z set_indexr5 )r r6 r+ r r Úgotr r r r Útest_unicode_encodingS s
|
r; zdata,kwargs,expectedza
|
1Úar z"a"
|
1Ú quotecharú"zb
|
1ÚnamesÚbÚ1ú
|
1T)r? Úskip_blank_linesFc sx | }d d fdd}|j dkrN|dkrN| dd¡rN|j tjjd d
|
¡ |j||fd i|}t ||¡ d S ) Nu r c s | ¡}t|S )N)r r )Ú_dataZbom_data©Úbomr- r r Ú_encode_data_with_bomx s z,test_utf8_bom.<locals>._encode_data_with_bomZpyarrowrB rC Tz Pyarrow can't read blank lines)Úreasonr )
|
ÚengineÚgetÚnodeZ
|
add_markerÚpytestÚmarkZxfailr r r )r r r, r Úrequestr rG r r rE r Ú test_utf8_bom_ s ÿþ
|
ý ÿrO c C sL t dgdgd}| }| |¡}d |¡}|jt||d}t ||¡ d S )Ng333333@Útest)Zmb_numZ multibytezmb_num,multibyte
|
4.8,testr$ )r Úformatr r r r r )r Ú utf_valueÚ encoding_fmtr r r r r r r r Útest_read_csv_utf_aliases s
|
|
rT zfile_path,encoding))Úior Úcsvz test1.csvr ))rU r r r8 r9 ))rU r r zsauron.SHIFT_JIS.csvÚshiftjisc
|
C sÐ | }|| }|j ||d}t||d}| |¡}|jr<tW 5 Q R X t ||¡ t|dd } |j | |d}| jrxtW 5 Q R X t ||¡ t|ddd } |j | |d}| jr¶tW 5 Q R X t ||¡ d S )Nr$ Úrb©Úmoder )rZ Ú buffering)r r) Úclosedr5 r r )
|
r Ú file_pathr Údatapathr Zfpathr Úfar Zfbr r r Ú test_binary_mode_file_buffers s
|
r` Ú pass_encodingc C sr | }| |¡}tddgi}tjd|dd<}| d¡ | d¡ |j||rP|nd d}t ||¡ W 5 Q R X d S ) NZfooÚbarzw+T)rZ r Zreturn_filelikezfoo
|
barr r$ )rQ r r r( r* Úseekr r ) r rR rS ra r r r r/ r r r r Útest_encoding_temp_file¶ s
|
|
|
rd c C s~ | }d}d}d}t ||gi}t ¡ N}| | d| |¡¡ | d¡ |j||d}t ||¡ |j rpt
|
W 5 Q R X d S )Nz shift-jisu ã¦ãã¨u ããÚ
|
r r$ ) r ÚtempfileÚNamedTemporaryFiler* r rc r r r r\ r5 )r r r Útitler r r/ r r r r Ú test_encoding_named_temp_fileÇ s
|
|
ri r z utf-16-bez utf-16-lezutf-32c C sR d}t | | ¡}t|d| d}tddgddgdd ggd
|
d gd }t ||¡ d S ) Nu a b
|
ï¼foo 0
|
bar 1
|
baz 2r )Ú delimiterr u ï¼foor rb r Zbazr" r< r@ )r r r )r r Z encoded_datar r r r r Ú%test_parse_encoded_special_charactersÜ s
|
"rk r r9 c C sp | }t ddddgddddgd d
|
d d gd }t ¡ &}|j|d|d |j||dd}W 5 Q R X t ||¡ d S )NZRaphaelZ Donatelloz Miguel AngelZLeonardoÚredZpurpleZorangeÚblueZsaizbo staffZnunchunkZkatana)ÚnameÚmaskZweaponF)Úindexr T)r Ú
|
memory_map)r r r( Úto_csvr r )r r r r ÚfileÚdfr r r Útest_encoding_memory_mapê s
|
|
|
ýÿ
|
ru c C sh | }t dgd d}d|jd< t d¡*}|j|dddd |j|d
|
d d d }W 5 Q R X t ||¡ d
|
S )zO
|
Chunk splits a multibyte character with memory_map=True
|
|
GH 43540
|
Zaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai )r u aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaÄ
iÿ zbug-gh43540.csvFr ©rp r r NTÚc)r rq rI )r Zilocr r( rr r r )r r rt ÚfnameÚdfrr r r Ú test_chunk_splits_multibyte_charü s
|
rz c C sÖ g }d}d}d}t t|t||D ]X}d dd t ||d D ¡d }z| d¡ W n tk
|
rp Y q$Y nX | |¡ q$| }t|}t d ¡,} |j | d
|
d
|
dd |j
|
| d d ddd}
|
W 5 Q R X t ||
|
¡ d S )zg
|
GH 43787
|
|
Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
|
é ú u ðÚ c S s g | ] }t |qS r )Úchr)Ú.0rw r r r Ú
|
<listcomp> s z,test_readcsv_memmap_utf8.<locals>.<listcomp>re r z utf8test.csvFrv NTrw )r rq rI r ) ÚrangeÚordr3 r ÚUnicodeEncodeErrorÚappendr r r( rr r r ) r ÚlinesZ line_lengthZ
|
start_charZend_charÚlnumÚliner rt rx ry r r r Útest_readcsv_memmap_utf8 s. "
|
ÿr rZ zw+bzw+tc C sh | }d}d|krd}t j|d$}| |¡ | d¡ | |¡}W 5 Q R X tg dgd}t ||¡ d S )Ns abcdÚtZabcdrY r r )rf ÚSpooledTemporaryFiler* rc r r r r )r rZ r ÚcontentÚhandlert r r r r Útest_not_readable. s
|
|
r )%Ú__doc__rU r r r2 rf r& ÚnumpyÚnprL Zpandasr r Zpandas._testingZ_testingr rM Z usefixturesZ skip_pyarrowZ xfail_pyarrowr r Z parametrizer0 r7 r; ÚnanrO rT r` rd ri rk ru rz r r r r r r Ú<module> sz
|
|
ýöþ
|
ýþ
|
|
ÿ
|
|
|
|
|
