U
|
¬ý°d¢H ã @ s´ d Z ddlmZ ddlZddlZddlmZmZ ddlZddlm Z ddl
|
m Z ddl m Z mZmZmZmZ ddlmZ dd lmZ dd
|
lmZ dd lmZ dd lmZ ddl Z dd l m Z m Z ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z) dddddZ*d8dddddd d d d Z+G d d dZ,G d!d" d"e,Z-G d#d$ d$e,Z.ee!d% d&d9d)d*dd+d,dd-d.d/d0d1Z/ee!d% d&d'dde j0e j0fd2dd-dd3d4d)d5d6d7Z1dS ):z parquet compat é )Ú annotationsN)ÚAnyÚLiteral)Úcatch_warnings)Úlib)Ú DtypeBackendÚFilePathÚ
|
ReadBufferÚStorageOptionsÚ WriteBuffer)Úimport_optional_dependency©ÚAbstractMethodError)Údoc)Úfind_stack_level)Úcheck_dtype_backend)Ú DataFrameÚ
|
get_option)Ú _shared_docs)ÚVersion)Ú IOHandlesÚ
|
get_handleÚ is_fsspec_urlÚis_urlÚstringify_pathÚstrÚBaseImpl)ÚengineÚreturnc C s¤ | dkrt d} | dkr|ttg}d}|D ]D}z | W S tk
|
rj } z|dt| 7 }W 5 d}~X Y q(X q(td| | dkrt S | dkrt S td dS )
|
zreturn our implementationÚautozio.parquet.engineÚ z
|
- NzÉUnable to find a usable engine; tried using: 'pyarrow', 'fastparquet'.
|
A suitable version of pyarrow or fastparquet is required for parquet support.
|
Trying to import the above resulted in these errors:ÚpyarrowÚ fastparquetz.engine must be one of 'pyarrow', 'fastparquet')r Ú PyArrowImplÚFastParquetImplÚ ImportErrorr Ú
|
ValueError)r Zengine_classesZ
|
error_msgsZ engine_classÚerr© r( úHd:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\pandas/io/parquet.pyÚ
|
get_engine, s$ $ÿ
|
r* ÚrbFz1FilePath | ReadBuffer[bytes] | WriteBuffer[bytes]r r
|
ÚboolzVtuple[FilePath | ReadBuffer[bytes] | WriteBuffer[bytes], IOHandles[bytes] | None, Any])ÚpathÚfsÚstorage_optionsÚmodeÚis_dirr c C s t | }t|r:|dkr:td}|jj|f|p0i \}}n |rVt|rN|dkrVtdd}|s|st|trt j
|
|¡st ||d|d}d}|j }|||fS )zFile handling for PyArrow.NÚfsspecr+ z8storage_options passed with buffer, or non-supported URLF©Zis_textr/ )r r r ÚcoreZ url_to_fsr r& Ú
|
isinstancer Úosr- Úisdirr Úhandle)r- r. r/ r0 r1 Úpath_or_handler2 Úhandlesr( r( r) Ú_get_path_or_handleN s8
|
ÿÿ
|
ÿþý
|
ü ÿr; c @ s> e Zd ZedddddZddddZd dd
|
d d Zd S )r r ÚNone)Údfr c C s t | tstdd S )Nz+to_parquet only supports IO with DataFrames)r5 r r& ©r= r( r( r) Úvalidate_dataframew s
|
zBaseImpl.validate_dataframer> c K s t | d S ©Nr )Úselfr= r- Ú compressionÚkwargsr( r( r) Úwrite| s zBaseImpl.writeN©r c K s t | d S r@ r )rA r- ÚcolumnsrC r( r( r) Úread s z BaseImpl.read)N)Ú__name__Ú
|
__module__Ú __qualname__Ú staticmethodr? rD rG r( r( r( r) r v s c @ sV e Zd ZddddZdddd d
|
d d dd ddZddejdfddd ddddZdS )r# r< rE c C s& t ddd dd l}dd l}|| _d S )Nr! z(pyarrow is required for parquet support.©Úextrar )r Zpyarrow.parquetZ(pandas.core.arrays.arrow.extension_typesÚapi)rA r! Úpandasr( r( r) Ú__init__ s ÿzPyArrowImpl.__init__ÚsnappyNr z FilePath | WriteBuffer[bytes]ú
|
str | Noneú bool | Noner
|
úlist[str] | None)r= r- rB Úindexr/ Úpartition_colsr c K s | |¡ d| dd ¡i}|d k r*||d< | jjj|f|} t|| dd ¡|d|d k d\}
|
} |d< t|
|
tjr¢t |
|
dr¢t|
|
j
|
t t fr¢|
|
j
|
}
|
t|
|
t r¢|
|
¡ }
|
zH|d k rÌ| jjj| |
|
f||d| n | jjj| |
|
fd|i| W 5 | d k rü| ¡ X d S ) NZschemaZpreserve_indexÚ
|
filesystemÚwb)r/ r0 r1 Úname)rB rV rB )r? ÚpoprN ZTableZ from_pandasr; r5 ÚioÚBufferedWriterÚhasattrrY r ÚbytesÚdecodeÚcloseÚparquetZwrite_to_datasetZ write_table) rA r= r- rB rU r/ rV rC Zfrom_pandas_kwargsÚtabler9 r: r( r( r) rD sT
|
|
|
û
|
ÿþý
|
þüû ÿÿÿ
|
zPyArrowImpl.writeFr, ú DtypeBackend | lib.NoDefault)Úuse_nullable_dtypesÚ dtype_backendr/ r c K sÚ d|d< i }|dkr2ddl m} | } | j|d< n|dkrDtj|d< td}
|
|
|
d kr\d|d
|
< t|| d d ¡|d d \} } |d < zD| j j
|
j | fd|i|} | j f |}|
|
d kr¼|j d dd}|W ¢S | d k rÔ| ¡ X d S )NTZuse_pandas_metadataZnumpy_nullabler )Ú_arrow_dtype_mappingZ types_mapperr! zmode.data_managerÚarrayZ split_blocksrW r+ )r/ r0 rF F)Úcopy)Zpandas.io._utilrf ÚgetÚpdZ
|
ArrowDtyper r; rZ r` rN ra Z
|
read_tableÚ to_pandasZ _as_manager)rA r- rF rd re r/ rC Zto_pandas_kwargsrf ÚmappingÚmanagerr9 r: Zpa_tableÚresultr( r( r) rG Ä s>
|
|
üÿÿÿ zPyArrowImpl.read)rQ NNN)rH rI rJ rP rD r Ú
|
no_defaultrG r( r( r( r) r# s ù 8úr# c @ sB e Zd ZddddZdddd dd
|
d d Zdd dd ddZdS )r$ r< rE c C s t ddd}|| _d S )Nr" z,fastparquet is required for parquet support.rL )r rN )rA r" r( r( r) rP ñ s
|
ÿzFastParquetImpl.__init__rQ Nr z,Literal[('snappy', 'gzip', 'brotli')] | Noner
|
)r= rB r/ r c s´ | |¡ d|kr"|d k r"tdd|kr4| d¡}|d k rDd|d< t|}t|rptd fdd|d< n r|td td
|
d $ | jj||f|||d | W 5 Q R X d S ) NÚ partition_onzYCannot use both partition_on and partition_cols. Use partition_cols for partitioning dataZhiveZ file_schemer2 c s j | dfpi ¡ S )NrX ©Úopen©r- Ú_©r2 r/ r( r) Ú<lambda> s
|
ÿÿz'FastParquetImpl.write.<locals>.<lambda>Ú open_withz?storage_options passed with file object or non-fsspec file pathT)Úrecord)rB Z write_indexrp ) r? r& rZ r r r r rN rD )rA r= r- rB rU rV r/ rC r( ru r) rD ù s8
|
|
ÿ
|
ÿ þûúzFastParquetImpl.write)r/ r c
|
s& i }| dd¡}| dtj¡}t| jjtdkr:d|d< |rFtd|tjk rXtdt|}d }t|r¸t d t| jjtd kr¤ j
|
|d
|
fpi j |d < qä fd d |d< n,t |t rätj |¡sät|d
|
dd}|j}z(| jj|f|} | jf d|i|W ¢S |d k r | ¡ X d S )Nrd Fre z0.7.1Z pandas_nullszNThe 'use_nullable_dtypes' argument is not supported for the fastparquet enginezHThe 'dtype_backend' argument is not supported for the fastparquet enginer2 z0.6.1r+ r. c s j | dfpi ¡ S )Nr+ rq rs ru r( r) rv E s
|
ÿÿz&FastParquetImpl.read.<locals>.<lambda>rw r3 rF )rZ r ro r rN Ú __version__r& r r r rr r. r5 r r6 r- r7 r r8 r` Z ParquetFilerk )
|
rA r- rF r/ rC Zparquet_kwargsrd re r: Z parquet_filer( ru r) rG ( sL ÿ
|
ÿ ÿÿ ÿ
|
zFastParquetImpl.read)rQ NNN)NN)rH rI rJ rP rD rG r( r( r( r) r$ ð s ù0 ÿr$ r/ )r/ r rQ r z$FilePath | WriteBuffer[bytes] | NonerR rS rT z bytes | None)r= r- r rB rU r/ rV r c
|
K sr t |tr|g}t|}|dkr(t ¡ n|} |j| | f||||d| |dkrjt | tjsbt| ¡ S dS dS )a}
|
Write a DataFrame to the parquet format.
|
|
Parameters
|
----------
|
df : DataFrame
|
path : str, path object, file-like object, or None, default None
|
String, path object (implementing ``os.PathLike[str]``), or file-like
|
object implementing a binary ``write()`` function. If None, the result is
|
returned as bytes. If a string, it will be used as Root Directory path
|
when writing a partitioned dataset. The engine fastparquet does not
|
accept file-like objects.
|
|
.. versionchanged:: 1.2.0
|
|
engine : {{'auto', 'pyarrow', 'fastparquet'}}, default 'auto'
|
Parquet library to use. If 'auto', then the option
|
``io.parquet.engine`` is used. The default ``io.parquet.engine``
|
behavior is to try 'pyarrow', falling back to 'fastparquet' if
|
'pyarrow' is unavailable.
|
compression : {{'snappy', 'gzip', 'brotli', 'lz4', 'zstd', None}},
|
default 'snappy'. Name of the compression to use. Use ``None``
|
for no compression. The supported compression methods actually
|
depend on which engine is used. For 'pyarrow', 'snappy', 'gzip',
|
'brotli', 'lz4', 'zstd' are all supported. For 'fastparquet',
|
only 'gzip' and 'snappy' are supported.
|
index : bool, default None
|
If ``True``, include the dataframe's index(es) in the file output. If
|
``False``, they will not be written to the file.
|
If ``None``, similar to ``True`` the dataframe's index(es)
|
will be saved. However, instead of being saved as values,
|
the RangeIndex will be stored as a range in the metadata so it
|
doesn't require much space and is faster. Other indexes will
|
be included as columns in the file output.
|
partition_cols : str or list, optional, default None
|
Column names by which to partition the dataset.
|
Columns are partitioned in the order they are given.
|
Must be None if path is not a string.
|
{storage_options}
|
|
.. versionadded:: 1.2.0
|
|
kwargs
|
Additional keyword arguments passed to the engine
|
|
Returns
|
-------
|
bytes if no path argument is provided else None
|
N)rB rU rV r/ )r5 r r* r[ ÚBytesIOrD ÚAssertionErrorÚgetvalue)
|
r= r- r rB rU r/ rV rC ÚimplZ path_or_bufr( r( r) Ú
|
to_parquetY s& <
|
þúù
|
r~ z FilePath | ReadBuffer[bytes]zbool | lib.NoDefaultrc )r- r rF r/ rd re r c K sb t |}|tjk r:d}|dkr&|d7 }tj|tt d nd}t| |j| f||||d|S )a
|
Load a parquet object from the file path, returning a DataFrame.
|
|
Parameters
|
----------
|
path : str, path object or file-like object
|
String, path object (implementing ``os.PathLike[str]``), or file-like
|
object implementing a binary ``read()`` function.
|
The string could be a URL. Valid URL schemes include http, ftp, s3,
|
gs, and file. For file URLs, a host is expected. A local file could be:
|
``file://localhost/path/to/table.parquet``.
|
A file URL can also be a path to a directory that contains multiple
|
partitioned parquet files. Both pyarrow and fastparquet support
|
paths to directories as well as file URLs. A directory path could be:
|
``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``.
|
engine : {{'auto', 'pyarrow', 'fastparquet'}}, default 'auto'
|
Parquet library to use. If 'auto', then the option
|
``io.parquet.engine`` is used. The default ``io.parquet.engine``
|
behavior is to try 'pyarrow', falling back to 'fastparquet' if
|
'pyarrow' is unavailable.
|
columns : list, default=None
|
If not None, only these columns will be read from the file.
|
|
{storage_options}
|
|
.. versionadded:: 1.3.0
|
|
use_nullable_dtypes : bool, default False
|
If True, use dtypes that use ``pd.NA`` as missing value indicator
|
for the resulting DataFrame. (only applicable for the ``pyarrow``
|
engine)
|
As new dtypes are added that support ``pd.NA`` in the future, the
|
output with this option will change to use those dtypes.
|
Note: this is an experimental option, and behaviour (e.g. additional
|
support dtypes) may change without notice.
|
|
.. deprecated:: 2.0
|
|
dtype_backend : {{"numpy_nullable", "pyarrow"}}, defaults to NumPy backed DataFrames
|
Which dtype_backend to use, e.g. whether a DataFrame should have NumPy
|
arrays, nullable dtypes are used for all dtypes that have a nullable
|
implementation when "numpy_nullable" is set, pyarrow is used for all
|
dtypes if "pyarrow" is set.
|
|
The dtype_backends are still experimential.
|
|
.. versionadded:: 2.0
|
|
**kwargs
|
Any additional kwargs are passed to the engine.
|
|
Returns
|
-------
|
DataFrame
|
zYThe argument 'use_nullable_dtypes' is deprecated and will be removed in a future version.TzFUse dtype_backend='numpy_nullable' instead of use_nullable_dtype=True.)Ú
|
stacklevelF)rF r/ rd re ) r* r ro ÚwarningsÚwarnÚ FutureWarningr r rG ) r- r rF r/ rd re rC r} Úmsgr( r( r) Ú read_parquet¬ s* A
|
ÿÿÿûúr )Nr+ F)Nr rQ NNN)2Ú__doc__Ú
|
__future__r r[ r6 Útypingr r r r Z pandas._libsr Zpandas._typingr r r r
|
r Zpandas.compat._optionalr Z pandas.errorsr Zpandas.util._decoratorsr Zpandas.util._exceptionsr Zpandas.util._validatorsr rO rj r r Zpandas.core.shared_docsr Zpandas.util.versionr Zpandas.io.commonr r r r r r* r; r r# r$ r~ ro r r( r( r( r) Ú<module> sR % û( mi ù R ú
|
