U
|
ý°dÝH ã @ s2 d Z ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddl Z ddl
|
m Z ddl Z ddl Z ddlZddlZddlmZ ddlZddlZddlZddlmZmZ ddlmZ dd lm Z ddl m m Z ddl!Z"dd
|
l!m#Z#m$Z$m%Z% ddl&m'Z( ddl)m m*Z+ dd l,m-Z-m.Z. ej/d d dd Z0dd Z1e j2 3e j2 4e5¡dd¡Z6e e j2 3e6dd¡¡Z7ej/e7ddd Z8ej9 :ddde;de<de =d¡eddd d ge<d >d d ¡e<d >d d ¡ddd
e ?d ¡j@d!d"d#e ?d ¡j@d!d$d#e ?d ¡j@d!d"d#dd
ddd
f g ¡d%d& ZAd'd( ZBd)d* ZCd+d, ZDej9 :d-ejEeCd.d/ejEe"jFd0d/ejEeje"jFe jGd1d2d/ejEeje"jFd3d1d4d/ejEeje"jFd5d1d6d/g¡ej9 :d7e"jFeCg¡d8d9 ZHd:d; ZId<d= ZJej/d>d? ZKG d@dA dAZLG dBdC dCZMej9 :dDdEgdFe#dGdHdIgfdJe"jN OdGdHdIgdKd d"gg¡fg¡dLdM ZPdNdO ZQej9 :dPdQdRdSg¡dTdU ZRe SdV¡dWdX ZTG dYdZ dZejUZVd[d\ ZWd]d^ ZXd_d` ZYdadb ZZej9 :dcdddedce dfdgdg¡dhg¡didj Z[dkdl Z\dmdn Z]e j^dodp Z_ej9 :dqe j`e jGg¡drds Zadtdu ZbdS )va<
|
manage legacy pickle tests
|
|
How to add pickle tests:
|
|
1. Install pandas version intended to output the pickle.
|
|
2. Execute "generate_legacy_storage_files.py" to create the pickle.
|
$ python generate_legacy_storage_files.py <output_dir> pickle
|
|
3. Move the created pickle to "data/legacy_pickle/<version>" directory.
|
é )ÚarrayN)Úpartial)ÚPath)Úcatch_warnings)Ú get_lzma_fileÚis_platform_little_endian)Úimport_optional_dependency)Úflatten_buffer)ÚIndexÚSeriesÚ period_range)ÚDayÚMonthEndÚmodule)Zscopec
|
C s0 ddl m} t | W 5 Q R £ S Q R X d S )Nr ©Úcreate_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr r r © r úRd:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\pandas/tests/io/test_pickle.pyÚcurrent_pickle_data8 s r c C s t |trt || ¡ d S | d¡r2t | |¡ nP|dkrb|tjkrT| tjks`tq| |kstn t td| dtj
|
}|| | d S )NZsp_Ú timestampZassert_Z_equal) Ú
|
isinstancer
|
ÚtmÚassert_index_equalÚ
|
startswithZ assert_equalÚpdZNaTÚAssertionErrorÚgetattrZassert_almost_equal)ÚresultÚexpectedÚtypZ
|
comparatorr r r Úcompare_elementD s
|
|
|
r ÚdataÚ legacy_pickleÚ*z*.pickle)Úparamsc C s
|
|| j S ©N)Úparam)ÚrequestÚdatapathr r r r" Y s s 123s 123456ÚIé é é ÚB)r, r+ é )r, é ÚC)ÚorderÚFc C s t | }t| d¡}||ks"tt| ttfr>|| kstnBt|tr|jdksVt|jdksdt|j snt|j
|
|j fkstd S )NÚAr* r- ) r Ú
|
memoryviewÚtobytesr r ÚbytesÚ bytearrayÚndimÚformatÚ
|
contiguousÚshapeÚnbytes)r! r r r r r Útest_flatten_bufferc s
|
|
r= c C sø t st d¡ t | ¡}| ¡ D ]Î\}}| ¡ D ]º\}}|| | }|dkr2|dkr2t ||¡ |jj |jj ks~t
|
|jj j rt
|
t |dk|dk¡ |jj }|t d t dksÀt
|
|tj dd }t|tj sàt
|
|tj dddksöt
|
|tj dd }t|tj st
|
|tj ddd
|
ksðt
|
q4|d kr |d kr t ||¡ t|j tsdt
|
|j t ksvt
|
|jd kst
|
t | d¡| d¡¡ q4|dkrÂ|dkrÂt ||¡ q4|dkrä|dkrät ||¡ q4t||| q4q"d S )Nz"known failure on non-little endianZseriesÚtsr r* r+ )Úhours)Údaysr? )Ú nanoseconds)r@ rA ÚindexZperiodÚM)Zdt_tzÚcatÚframe)Z dt_mixed_tzsZ
|
cat_onecolZ cat_and_float)r ÚpytestÚskipr Ú read_pickleÚitemsr Úassert_series_equalrB Úfreqr Ú normalizer Z Timedeltar r r ZfreqstrÚshiftÚassert_frame_equalr ) r" r! r ÚdvÚdtr r rK Úresr r r Ú test_pickles s<
|
|
rR c C s* t |d}tj| |dd W 5 Q R X d S ©NÚwbéÿÿÿÿ©Úprotocol©ÚopenÚpickleÚdump©ÚobjÚpathÚfhr r r Úpython_pickler® s r` c
|
C s6 t | d"}| d¡ t |¡W 5 Q R £ S Q R X d S )NÚrbr )rY ÚseekrZ Úload)r^ r_ r r r Úpython_unpickler³ s
|
rd Ú pickle_writerÚpython)ÚidZpandas_proto_defaultrV Zpandas_proto_highestr/ Zpandas_proto_4é Zpandas_proto_5Úwriterc C sØ | }| ¡ D ]Æ\}}| ¡ D ]´\}}t ¡ }||| t |¡} t| || t|} t| || t|dd}
|
||| |
|
d¡ W 5 Q R X t|dd}
|
t |
|
¡} |
|
d¡ W 5 Q R X t| || W 5 Q R X q q d S )NrT ©Úmoder ra ) rI r Ú ensure_cleanr rH r rd rY rb ) r re ri r! r rO rP r r^ r Úhandler r r Útest_round_trip_current¹ s
|
|
|
|
|
rn c C s( t ¡ } t | jtj¡}t | |¡ d S r% )r Ú makeDataFrameÚround_trip_pathlibÚ to_pickler rH rN ©Údfr r r r Útest_pickle_path_pathlibã s rt c C s( t ¡ } t | jtj¡}t | |¡ d S r% )r ro Zround_trip_localpathrq r rH rN rr r r r Útest_pickle_path_localpathé s ru c C s dt ¡ dS )NÚ__z __.pickle)ÚuuidÚuuid4r r r r Úget_random_pathô s ry c @ sX e Zd ZejZdd Zdd Zej
|
ddddd g¡d
|
d Z d d Z dd Z dd ZdS )ÚTestCompressionc
|
C s` |d krt ||¡ d S |dkr.t |d¡}nð|dkrDt |d¡}nÚ|dkrtj|dtjd}| |t
|
j |¡¡ W 5 Q R X n|dkrÚt|d@}t j|dd&}| |t
|
j |¡¡}| ||¡ W 5 Q R X W 5 Q R X nD|d krðt |d}n.|d
|
kr td |d ¡}nd | }t||dkr\t|d$}| | | ¡ ¡ W 5 Q R X W 5 Q R X d S )NÚgzipÚwÚbz2Úzip©Ú compressionÚtarra rj ÚxzZzstdZ zstandardrT z Unrecognized compression type: )r~ r )ÚshutilÚcopyfiler{ rY r} ÚBZ2FileÚzipfileÚZipFileÚ ZIP_DEFLATEDÚwriteÚosr^ ÚbasenameÚtarfileÚ
|
gettarinfoÚaddfiler r Ú
|
ValueErrorÚread) ÚselfZsrc_pathÚ dest_pathr Úfr_ r ÚtarinfoÚmsgr r r Ú compress_fileü s2 "
|
|
|
z TestCompression.compress_filec C s² |}|d }|d }t |¡}t |¡t}t ¡ }|j||d t j||d*} t|d}
|
|
|
| ¡ ¡ W 5 Q R X W 5 Q R X tj |d d} t
|
|| ¡ W 5 Q R X W 5 Q R X d S )Nú .compressedú.rawr rT ) r rl ro rq Údecompress_filerY r r r rH rN ) r r ry ÚbaseÚpath1Úpath2Úp1Úp2rs r r_ Údf2r r r Útest_write_explicit s "z#TestCompression.test_write_explicitr Ú ÚNoneÚbadZ7zc
|
C sJ t jtdd2 t |¡ }t ¡ }|j||d W 5 Q R X W 5 Q R X d S )Nz Unrecognized compression type)Úmatchr )rF Zraisesr r rl ro rq )r r ry r^ rs r r r Útest_write_explicit_bad/ s z'TestCompression.test_write_explicit_badc C s¾ |}|| }|d }| j | ¡ ¡}t |¡}t |¡p}t ¡ } | |¡ tj||d*}
|
t|d} | |
|
|
¡ ¡ W 5 Q R X W 5 Q R X t j |d d} t | | ¡ W 5 Q R X W 5 Q R X d S )Nr r rT )Ú_extension_to_compressionÚgetÚlowerr rl ro rq r rY r r r rH rN ) r Úcompression_extry r r r r r r rs r r_ r r r r Útest_write_infer6 s
|
"z TestCompression.test_write_inferc
|
|
C s |}|d }|d }t |¡\}t |¡F}t ¡ }|j|d d | j|||d tj||d} t || ¡ W 5 Q R X W 5 Q R X d S )Nr r r )r rl ro rq r r rH rN )
|
r r ry r r r r r rs r r r r Útest_read_explicitL s z"TestCompression.test_read_explicitc
|
C s |}|d }|| }| j | ¡ ¡}t |¡X}t |¡B}t ¡ } | j|d d | j|||d t |¡}
|
t
|
| |
|
¡ W 5 Q R X W 5 Q R X d S )Nr r ) r¦ r§ r¨ r rl ro rq r r rH rN ) r r© ry r r r r r r rs r r r r Útest_read_infer^ s
|
z TestCompression.test_read_inferN)Ú__name__Ú
|
__module__Ú __qualname__ÚicomZextension_to_compressionr¦ r r rF ÚmarkÚ parametrizer¥ rª r« r¬ r r r r rz ù s
|
rz c @ s* e Zd Zej dddddg¡dd ZdS ) Ú TestProtocolrW rU r r* r+ c C sF t |¡2}t ¡ }|j||d t |¡}t ||¡ W 5 Q R X d S )NrV ©r rl ro rq r rH rN )r rW ry r^ rs r r r r Ú test_readx s
|
|
zTestProtocol.test_readN)r r® r¯ rF r± r² rµ r r r r r³ w s r³ Ú pickle_fileÚexcolsz test_py27.pklÚaÚbÚcztest_mi_py27.pklr3 c C s* | ddd|}t |¡}t |j|¡ d S )NÚior! rZ )r rH r r Úcolumns)r( r¶ r· r^ rs r r r Útest_unicode_decode_error s
|
r½ c
|
C sl t ¡ Z} t ¡ }t| d}| |¡ W 5 Q R X t| d}t |¡}W 5 Q R X t ||¡ W 5 Q R X d S )NrT ra )r rl ro rY rq r rH rN )r^ rs r_ r r r r Ú test_pickle_buffer_roundtrip s
|
r¾ Úmockurlzhttp://url.comzftp://test.comzhttp://gzip.comc sp dd }G dd d t ¡ H fdd}t ¡ }|| | d|¡ t |¡}t ||¡ W 5 Q R X d S )Nc S s* t |d}tj| |dd W 5 Q R X d S rS rX r\ r r r r` s z3test_pickle_generalurl_read.<locals>.python_picklerc @ s: e Zd ZddddZdd Zdd Zd d
|
Zd d ZdS ) z5test_pickle_generalurl_read.<locals>.MockReadResponseN©Úreturnc S s. t |d| _d|kr ddi| _n
|
ddi| _d S )Nra r{ zContent-Encodingr¡ )rY ÚfileÚheaders)r r^ r r r Ú__init__² s z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c S s | S r% r ©r r r r Ú __enter__¹ s z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c W s | ¡ d S r% )Úclose)r Úargsr r r Ú__exit__¼ s z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c S s
|
| j ¡ S r% )r r rÅ r r r r ¿ s z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc S s
|
| j ¡ S r% )r rÇ rÅ r r r rÇ Â s z;test_pickle_generalurl_read.<locals>.MockReadResponse.close)r r® r¯ rÄ rÆ rÉ r rÇ r r r r ÚMockReadResponse± s
|
rÊ c s S r% r )rÈ Úkwargs©rÊ r^ r r Úmock_urlopen_readÇ s z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r rl ro Úsetattrr rH rN )Z monkeypatchr¿ r` rÍ rs r r rÌ r Útest_pickle_generalurl_read© s
|
|
|
rÏ Zfsspecc C sD t ¡ 2 d} t ¡ }| | ¡ t | ¡}t ||¡ W 5 Q R X d S )Nzmemory://afiler´ )r¿ rs r r r r Ú test_pickle_fsspec_roundtripÑ s
|
|
|
rÐ c @ s e Zd ZddddZdS )ÚMyTzNrÀ c C s d S r% r rÅ r r r rÄ Ü s z MyTz.__init__)r r® r¯ rÄ r r r r rÑ Û s rÑ c C sD t tdt f} t | ¡}t |d | d ¡ t|d ts@td S )N)Zdtyper r* )r ÚobjectrÑ r Úround_trip_picklerJ r r )r r r r r Ú test_read_pickle_with_subclassà s
|
rÔ c C s t ¡ }t ¡ }|j|| d t| ¡ }W 5 Q R X t ¡ }|j|| d | d¡ | ¡ |ksn| dksnt
|
t j || d}| d¡ t ||¡ dS )ze
|
Read/write from binary file-objects w/wo compression.
|
|
GH 26237, GH 29054, and GH 29570
|
r r )r{ r~ r N)r ro rl rq r Ú
|
read_bytesr» ÚBytesIOrb Úgetvaluer r rH rN )r rs r^ Ú referenceÚbufferZread_dfr r r Ú%test_pickle_binary_object_compressioné s
|
|
|
rÚ c C s8 | }|}dd }|| ||j || ||j d S )Nc S s t | ¡}t | |¡ d S r% )r rÓ rN )rE Ú unpickledr r r Ú_test_roundtrip s
|
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)ÚT)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdrE rÜ r r r Ú+test_pickle_dataframe_with_multilevel_index s
|
rÞ c C sB t dddd} ttj t| ¡| }t |¡}|jj dks>t
|
d S )Nz1/1/2011z1/1/2012rC )rK ) r r ÚnpÚrandomZrandnÚlenr rÓ rB rK r )Zprngr> Znew_tsr r r Ú"test_pickle_timeseries_periodindex s
|
râ Únamei g H@iÑ é )r* r+ c C s$ t t j| d¡}|j| ks td S )N)rã )r rÓ ZmakeTimeSeriesrã r )rã rÛ r r r Útest_pickle_preserve_name s rå c C s t | ¡}t || ¡ d S r% ©r rÓ rJ )Zdatetime_seriesZunp_tsr r r Útest_pickle_datetimes% s
|
rç c C s t | ¡}t || ¡ d S r% ræ )Z string_seriesZ
|
unp_seriesr r r Útest_pickle_strings* s
|
rè c C sf t td d¡jdg } t | ¡}|jjd jdks:t |jjd j
|
dksPt t |dg | ¡ d S )NÚabcÚcategoryr r* )r* T) r ÚlistZastypeZilocr rÓ Z_mgrÚblocksr8 r r; rJ )ZserrQ r r r Ú test_pickle_preserves_block_ndim/ s
|
|
rí rW c C s@ t td¡}t t|j| |dtt j|d¡}t ||¡ d S )Ni )rW r r ) r Ú DataFrameÚranger rp r rq rH rN )rW r rs r r r r Ú%test_pickle_big_dataframe_compression< s þrð c C sN t j tdd¡} t| d}t |¡}W 5 Q R X tjg g d}t
|
||¡ d S )Nz1.2.4z empty_frame_v1_2_4-GH#42345.pklra )rB r¼ ) r r^ ÚjoinÚlegacy_dirnamerY rZ rc r rî r rN )r^ Úfdrs r r r r Ú#test_pickle_frame_v124_unpickle_130G s
|
rô )cÚ__doc__r r} ÚdatetimeÚ functoolsr Úglobr{ r» r Úpathlibr rZ r r rw Úwarningsr r Únumpyrß rF Z pandas.compatr r Zpandas.compat._optionalr Zpandas.compat.compressorsr Z pandas.util._test_decoratorsÚutilZ_test_decoratorsÚtdZpandasr r
|
r r Zpandas._testingZ_testingr Zpandas.io.commonÚcommonr° Zpandas.tseries.offsetsr r Zfixturer r r^ rñ ÚdirnameÚ__file__rò Úfilesr" r± r² r7 r4 Ú PickleBufferÚcastZarangeZreshaper= rR r` rd r& rq ÚHIGHEST_PROTOCOLrn rt ru ry rz r³ Z
|
MultiIndexZ from_arraysr½ r¾ rÏ Z
|
skip_if_norÐ ÚtzinforÑ rÔ rÚ rÞ râ rå rç rè Z skip_array_manager_invalid_testrí ÚDEFAULT_PROTOCOLrð rô r r r r Ú<module> sÜ
|
|
|
|
$õþ
|
. þþøþ
|
~
|
þþþ
|
|
ÿ
|
%
|
ÿ
|
|
|
