U
|
¬ý°d\: ã @ s$ d dl mZ d dlmZ d dlmZmZmZmZ d dl Z d dl
|
Z d dl m Z mZ d dlmZmZmZmZ d dlmZ d dlmZ d d lmZ d d
|
lmZm Z d d l m Z m Z d d l m!Z! d d l"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z( erðd dl)m*Z*m+Z+ G dd de&Z,dddddZ-dddddZ.dS ) é )Ú annotations)Ú defaultdict)Ú TYPE_CHECKINGÚHashableÚMappingÚSequenceN)ÚlibÚparsers)Ú ArrayLikeÚDtypeArgÚDtypeObjÚ ReadCsvBuffer)Úimport_optional_dependency)Ú DtypeWarning)Úfind_stack_level)Úis_categorical_dtypeÚ pandas_dtype)Ú concat_compatÚunion_categoricals)Úensure_index_from_sequences)Ú dedup_namesÚis_potential_multi_index)Ú
|
ParserBaseÚ ParserErrorÚ is_index_col)ÚIndexÚ
|
MultiIndexc s e Zd ZU ded< ded< ddd fdd Zdd
|
d d Zdd
|
d dZd dddddZdddddZdd Z d!d dd d d Z
|
Z S )"ÚCParserWrapperÚboolÚ
|
low_memoryzparsers.TextReaderÚ_readerzReadCsvBuffer[str]ÚNone)ÚsrcÚreturnc s t |¡ || _| ¡ }| dd¡| _| jdk |d< | j|d< | jj |d< dD ]}| |d ¡ qPt
|
| dd ¡|d< d|ks|d t j krd |d< |d d
|
kr¨td
|
tj|f|| _| jj| _| jd k}| jjd kràd | _n"| | jj| j|¡\| _| _| _}| jd kr tt| jj| _| jd d
| _| jrÔ| | j| j¡ | jd k sXt | j d krt | j¡s| ! | j¡ t"| jt" kr² fd d t#| jD | _t"| jt" k rÔ| ! | j¡ | $| j¡ | %¡ | j| _| j&sz| jj'dkrDt(| jrDd| _)| *| j| j¡\}| _| _| jd krD|| _| jjd krz|sz| jd k sht d gt"| j | _| jj'dk| _+d S )Nr FZallow_leading_colsÚusecolsÚ on_bad_lines)Zstorage_optionsÚencodingZ
|
memory_mapÚ compressionÚdtypeZ dtype_backendÚnumpyZpyarrowÚstringc s$ g | ] \}}| ks | kr|qS © r+ )Ú.0ÚiÚn©r$ r+ úYd:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\pandas/io/parsers/c_parser_wrapper.pyÚ
|
<listcomp> s ýz+CParserWrapper.__init__.<locals>.<listcomp>r T),ÚsuperÚ__init__ÚkwdsÚcopyÚpopr Ú index_colr$ r% ÚvalueÚensure_dtype_objsÚgetr Z
|
no_defaultr r Z
|
TextReaderr Z unnamed_colsÚnamesÚheaderZ _extract_multi_indexer_columnsÚ index_namesÚ col_namesÚlistÚrangeZ table_widthÚ
|
orig_namesÚ_evaluate_usecolsÚAssertionErrorZ usecols_dtypeÚsetÚissubsetZ_validate_usecols_namesÚlenÚ enumerateZ _validate_parse_dates_presenceÚ_set_noconvert_columnsÚ_has_complex_date_colÚ leading_colsr Z_name_processedÚ_clean_index_namesZ_implicit_index)Úselfr" r4 ÚkeyZ passed_namesr= ©Ú __class__r/ r0 r3 < s ÿ
|
|
|
ýû ÿ
|
ýþ ÿüû zCParserWrapper.__init__)r# c C s( z| j ¡ W n tk
|
r" Y nX d S ©N)r ÚcloseÚ
|
ValueError)rL r+ r+ r0 rQ Â s zCParserWrapper.closec s^ | j dk stdd t| j D fdd| jD }| || j¡}|D ]}| j |¡ qHdS )z¹
|
Set the columns that should not undergo dtype conversions.
|
|
Currently, any column that is involved with date parsing will not
|
undergo such conversions.
|
Nc S s i | ]\}}||qS r+ r+ )r, r- Úxr+ r+ r0 Ú
|
<dictcomp>Ô s z9CParserWrapper._set_noconvert_columns.<locals>.<dictcomp>c s g | ] } | qS r+ r+ ©r, rS ©Z
|
names_dictr+ r0 r1 Õ s z9CParserWrapper._set_noconvert_columns.<locals>.<listcomp>)rA rC rG r; Z _set_noconvert_dtype_columnsr Z set_noconvert)rL Z col_indicesZnoconvert_columnsÚcolr+ rV r0 rH É s þz%CParserWrapper._set_noconvert_columnsNz
|
int | Nonez_tuple[Index | MultiIndex | None, Sequence[Hashable] | MultiIndex, Mapping[Hashable, ArrayLike]])Únrowsr# c
|
s¶ z,| j r | j |¡}t|}n | j |¡}W nª tk
|
rÖ | jrÈd| _t| jt | j| j
|
}| j || j
|
| j | j d¡d\} }| | j¡ | jd k r¤| ¡ fdd| ¡ D }| |f Y S | ¡ Y nX d| _| j}| jjr| jrütdg }| j
|
r<| jjt| j
|
kr<tdt| j
|
d| jj d t| jjD ]H}| j
|
d krd| |¡} n| | j
|
| ¡} | j | |d
|
d } | | ¡ qHt |}| jd k r°| |¡}t|t || j
|
}t | ¡ }
|
d d t!||
|
D }| "||¡\} } | | | j¡} nªt | ¡ }
|
| jd k s t#t$| j}t|t || j
|
}| jd k rP| |¡}d d |
|
D } | jd krv| %|| ¡ dd t!||
|
D }| "||¡\}} | &| | |¡\}} || | fS )NFr( ©r( c s i | ]\}}| kr||qS r+ r+ )r, ÚkÚv©Úcolumnsr+ r0 rT s z'CParserWrapper.read.<locals>.<dictcomp>z file structure not yet supportedz,Could not construct index. Requested to use z number of columns, but z left to parse.T)Útry_parse_datesc S s i | ]\}\}}||qS r+ r+ ©r, rZ r- r[ r+ r+ r0 rT 0 s
|
c S s g | ] }|d qS )é r+ rU r+ r+ r0 r1 H s z'CParserWrapper.read.<locals>.<listcomp>c S s i | ]\}\}}||qS r+ r+ r_ r+ r+ r0 rT L s
|
)'r r Zread_low_memoryÚ_concatenate_chunksÚreadÚ StopIterationZ _first_chunkr rA r r7 Z_get_empty_metar= r4 r: Z _maybe_make_multi_index_columnsr> r$ Ú_filter_usecolsÚitemsrQ r; rJ rI ÚNotImplementedErrorrF r r@ r6 Ú_maybe_parse_datesÚappendr ÚsortedÚzipZ_do_date_conversionsrC r? Z_check_data_lengthZ _make_index)rL rX ÚchunksÚdatar; ÚindexZcol_dictZarraysr- ÚvaluesZ data_tupsZ column_namesZ date_dataZalldatar+ r\ r0 rb Þ s
|
|
þ
|
ü
|
|
|
ÿ
|
ÿ
|
|
zCParserWrapper.readzSequence[Hashable])r; r# c s@ | | j|¡ d k r<t|t kr< fddt|D }|S )Nc s$ g | ] \}}| ks | kr|qS r+ r+ )r, r- Únamer/ r+ r0 r1 W s z2CParserWrapper._filter_usecols.<locals>.<listcomp>)rB r$ rF rG )rL r; r+ r/ r0 rd S s
|
ÿz CParserWrapper._filter_usecolsc C sH t | jjd }d }| jjdkr@| jd k r@| || j¡\}}| _||fS )Nr )r? r r< rJ r7 rK )rL r; Z idx_namesr+ r+ r0 Ú_get_index_names\ s ÿ z CParserWrapper._get_index_namesTÚint)rm r^ c C s4 |r0| |¡r0| j|| jd k r(| j| nd d}|S )N)rW )Z_should_parse_datesZ
|
_date_convr= )rL rn rm r^ r+ r+ r0 rg g s þz!CParserWrapper._maybe_parse_dates)N)T) Ú__name__Ú
|
__module__Ú __qualname__Ú__annotations__r3 rQ rH rb rd rp rg Ú __classcell__r+ r+ rN r0 r 8 s
|
þu r zlist[dict[int, ArrayLike]]Údict)rk r# c
|
sä t | d ¡ }g }i }|D ] fdd| D }dd |D }dd |D }| ¡ }t|rpt|dd| < q t|| < t|d kr | jt t ¡kr |
|
t ¡ q |ràd
|
|¡}d d | d g¡} t j| tt d |S )z±
|
Concatenate chunks of data read with low_memory=True.
|
|
The tricky part is handling Categoricals, where different chunks
|
may have different inferred categories.
|
r c s g | ]}| ¡qS r+ )r6 )r, Úchunk©ro r+ r0 r1 | s z'_concatenate_chunks.<locals>.<listcomp>c S s h | ]
|
}|j qS r+ rY )r, Úar+ r+ r0 Ú <setcomp>~ s z&_concatenate_chunks.<locals>.<setcomp>c S s h | ]}t |s|qS r+ )r rU r+ r+ r0 r{ s F)Zsort_categoriesr` ú,ú z Columns (zK) have mixed types. Specify dtype option on import or set low_memory=False.)Ú
|
stacklevel)r? Úkeysr6 r r r rF r( ÚnpÚobjectrh ÚstrÚjoinÚwarningsÚwarnr r )
|
rk r; Zwarning_columnsÚresultZarrsZdtypesZnon_cat_dtypesr( Z warning_namesZwarning_messager+ ry r0 ra p s*
|
|
ÿÿra z*DtypeArg | dict[Hashable, DtypeArg] | Nonez*DtypeObj | dict[Hashable, DtypeObj] | None)r( r# c sx t trHt ¡ t fdd} ¡ D ]}t| ||< q.|S t trdfddD S dk rttS S )zc
|
Ensure we have either None, a dtype object, or a dictionary mapping to
|
dtype objects.
|
c s S rP r+ r+ )Ú default_dtyper+ r0 Ú<lambda> ó z#ensure_dtype_objs.<locals>.<lambda>c s i | ]}|t | qS r+ )r )r, rZ rY r+ r0 rT ¤ s z%ensure_dtype_objs.<locals>.<dictcomp>N)Ú
|
isinstancer r Údefault_factoryr rw )r( Zdtype_convertedrM r+ )r r( r0 r9 s
|
|
r9 )/Ú
|
__future__r Ú collectionsr Útypingr r r r r r) r Z pandas._libsr r Zpandas._typingr
|
r r r Zpandas.compat._optionalr Z pandas.errorsr Zpandas.util._exceptionsr Zpandas.core.dtypes.commonr r Zpandas.core.dtypes.concatr r Zpandas.core.indexes.apir Zpandas.io.commonr r Z pandas.io.parsers.base_parserr r r Zpandasr r r ra r9 r+ r+ r+ r0 Ú<module> s* :%
|
