U
|
¬ý°dÅ ã @ sp d dl mZ d dlmZ d dlmZ d dlmZ d dlZ d dlm
|
Z
|
d dl m Z d dl mZ G d d
|
d
|
eZdS ) é )Ú annotations)Ú
|
ReadBuffer)Úimport_optional_dependency)Ú
|
is_integerN)Ú DataFrame)Ú_arrow_dtype_mapping)Ú
|
ParserBasec s\ e Zd ZdZddd fdd Zdd Zdd d
|
d Zd d d ddZd d ddZ Z S )ÚArrowParserWrapperz7
|
Wrapper for the pyarrow engine for read_csv()
|
zReadBuffer[bytes]ÚNone)ÚsrcÚreturnc s$ t |¡ || _|| _| ¡ d S )N)ÚsuperÚ__init__Úkwdsr Ú _parse_kwds)Úselfr r ©Ú __class__© ú]d:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\pandas/io/parsers/arrow_parser_wrapper.pyr s zArrowParserWrapper.__init__c C sf | j d¡}|dkrdn|| _| | j d ¡\| _| _| j d }t|trRtdt | j d | _
|
dS )z?
|
Validates keywords before passing to pyarrow.
|
ÚencodingNzutf-8ÚusecolsÚ na_valuesz?The pyarrow engine doesn't support passing a dict for na_values) r Úgetr Z_validate_usecols_argr Z usecols_dtypeÚ
|
isinstanceÚdictÚ
|
ValueErrorÚlistr )r r r r r r r s ÿ
|
|
ÿz ArrowParserWrapper._parse_kwds)r c C sª dddddd}| ¡ D ]4\}}|| jkr| j |¡dk r| j |¡| j|< qdd | j ¡ D | _d
|
d | j ¡ D | _| jdk| jdk r| jn| jd | jd | _dS ) z:
|
Rename some arguments to pass to pyarrow
|
Úinclude_columnsÚ null_valuesÚ escape_charÚignore_empty_linesÚ decimal_point)r r Ú
|
escapecharZskip_blank_linesÚdecimalNc S s& i | ] \}}|d k r|dkr||qS )N)Ú delimiterÚ
|
quote_charr r! r ©Ú.0Z option_nameZ option_valuer r r Ú
|
<dictcomp>; s ÿý z;ArrowParserWrapper._get_pyarrow_options.<locals>.<dictcomp>c S s& i | ] \}}|d k r|dkr||qS )N)r r Z true_valuesZ false_valuesr" r r' r r r r) B s ÿý Zskiprows)Zautogenerate_column_namesZ skip_rowsr ) Úitemsr r ÚpopÚ parse_optionsÚconvert_optionsÚheaderr Ú read_options)r ÚmappingZ pandas_nameZ pyarrow_namer r r Ú_get_pyarrow_options, s* ûþþÿûz'ArrowParserWrapper._get_pyarrow_optionsr )Úframer c
|
C sj t |j}d}| jdkrn| jdkr6| jdkr6t|| _t | j|krftt|t | j | j | _d}| j|_| |j|¡\|_}| jdk rt| jD ]<\}}t |rº|j| | j|< q||jkrt
|
d| dq|j | jddd | jdkr|sdgt |j j |j _| j d¡dk rfz| | j d¡¡}W n, tk
|
rd } z t
|
|W 5 d}~X Y nX |S )z÷
|
Processes data read in based on kwargs.
|
|
Parameters
|
----------
|
frame: DataFrame
|
The DataFrame to process.
|
|
Returns
|
-------
|
DataFrame
|
The processed DataFrame.
|
TNFzIndex z invalid)ZdropZinplaceZdtype)ÚlenÚcolumnsr. ÚnamesÚranger Z_do_date_conversionsZ index_colÚ enumerater r Z set_indexÚindexr r ZastypeÚ TypeError)r r2 Znum_colsZmulti_index_namedÚiÚitemÚer r r Ú_finalize_pandas_outputW s4
|
|
|
|
|
|
z*ArrowParserWrapper._finalize_pandas_outputc C s t d}| ¡ |j| j|jf | j|jf | j|jf | j d}| j
|
d dkr`|j t j d}n(| j
|
d dkr|j t jd}n| ¡ }| |¡S )zÿ
|
Reads the contents of a CSV file into a DataFrame and
|
processes it according to the kwargs passed in the
|
constructor.
|
|
Returns
|
-------
|
DataFrame
|
The DataFrame created from the CSV file.
|
z pyarrow.csv)r/ r, r- Z dtype_backendZpyarrow)Z types_mapperZnumpy_nullable)r r1 Zread_csvr Z ReadOptionsr/ Z ParseOptionsr, ZConvertOptionsr- r Z to_pandasÚpdZ
|
ArrowDtyper r r= )r Z pyarrow_csvÚtabler2 r r r Úread s üzArrowParserWrapper.read)
|
Ú__name__Ú
|
__module__Ú __qualname__Ú__doc__r r r1 r= r@ Ú __classcell__r r r r r s +3r )Ú
|
__future__r Zpandas._typingr Zpandas.compat._optionalr Z pandas.core.dtypes.inferencer Zpandasr> r Zpandas.io._utilr Z pandas.io.parsers.base_parserr r r r r r Ú<module> s
|
