U
|
¬ý°dçl ã @ s( d Z ddlmZ ddlmZ ddlmZmZ ddlZddlm Z ddl
|
Z ddl m Z mZmZ ddlmZmZ ddlZdd lmZmZ dd
|
lmZ dd lmZmZm Z m Z m Z dd l m Z m!Z! ddl"m# m$ m%Z& dd l'm(Z( dddddZ)ddddddZ*G dd dZ+G dd de(ej,Z-dS )a
|
Read SAS7BDAT files
|
|
Based on code written by Jared Hobbs:
|
https://bitbucket.org/jaredhobbs/sas7bdat
|
|
See also:
|
https://github.com/BioStatMatt/sas7bdat
|
|
Partial documentation of the file format:
|
https://cran.r-project.org/package=sas7bdat/vignettes/sas7bdat.pdf
|
|
Reference for binary data compression:
|
http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm
|
é )Ú annotations)Úabc)ÚdatetimeÚ timedeltaN)Úcast)ÚCompressionOptionsÚFilePathÚ
|
ReadBuffer)ÚEmptyDataErrorÚOutOfBoundsDatetime)Ú DataFrameÚisna)Ú
|
get_handle)Úread_double_with_byteswapÚread_float_with_byteswapÚread_uint16_with_byteswapÚread_uint32_with_byteswapÚread_uint64_with_byteswap)ÚParserÚget_subheader_index)Ú
|
ReaderBaseÚfloatÚstr©Z sas_datetimeÚunitc C sV t | rtjS |dkr,tdddt| d S |dkrJtdddt| d S tdd S )NÚsé¨ é )ÚsecondsÚd)Údayszunit must be 'd' or 's')r ÚpdZNaTr r Ú
|
ValueErrorr © r# úMd:\z\workplace\vscode\pyvenv\venv\Lib\site-packages\pandas/io/sas/sas7bdat.pyÚ_parse_datetime< s r% z pd.Series)Ú sas_datetimesr Úreturnc C sJ zt j| |ddW S tk
|
rD | jt|d}tt j|}| Y S X dS )aÉ
|
Convert to Timestamp if possible, otherwise to datetime.datetime.
|
SAS float64 lacks precision for more than ms resolution so the fit
|
to datetime.datetime is ok.
|
|
Parameters
|
----------
|
sas_datetimes : {Series, Sequence[float]}
|
Dates or datetimes in SAS
|
unit : {str}
|
"d" if the floats represent dates, "s" for datetimes
|
|
Returns
|
-------
|
Series
|
Series of datetime64 dtype or datetime.datetime.
|
z
|
1960-01-01)r Úorigin©r N)r! Ú to_datetimer Úapplyr% r ÚSeries)r& r Zs_seriesr# r# r$ Ú_convert_datetimesJ s r- c @ sX e Zd ZU ded< ded< ded< ded< ded< ded < ddddddd
|
d d d ZdS )Ú_ColumnÚintÚcol_idú str | bytesÚnameÚlabelÚformatÚbytesÚctypeÚlengthÚNone)r0 r2 r3 r4 r6 r7 r' c C s( || _ || _|| _|| _|| _|| _d S ©N)r0 r2 r3 r4 r6 r7 )Úselfr0 r2 r3 r4 r6 r7 r# r# r$ Ú__init__l s
|
z_Column.__init__N)Ú__name__Ú
|
__module__Ú __qualname__Ú__annotations__r; r# r# r# r$ r. d s
|
r. c @ sâ e Zd ZU dZded< ded< dSd d
|
d
|
d d d
|
d
|
d dd ddZddddZddddZddddZddddZ ddd d Z
|
d dd d Z ddd!d"d#Z dddd$d%d&Z ddd'd(d)Zddd*d+d,d-Zddd.d/Zd
|
dd0d1Zddd2d3Zddd4d5Zdddd+d6d7Zdddd+d8d9Zdddd+d:d;Zdddd+d<d=Zdddd+d>d?Zdddd+d@dAZdddd+dBdCZdddd+dDdEZdTd d dFdGdHZ dIdJ Z d ddKdLZ dMdN Z dOd*dPdQdRZ dS )UÚSAS7BDATReaderaË
|
Read SAS files in SAS7BDAT format.
|
|
Parameters
|
----------
|
path_or_buf : path name or buffer
|
Name of SAS file or file-like object pointing to SAS file
|
contents.
|
index : column identifier, defaults to None
|
Column to use as index.
|
convert_dates : bool, defaults to True
|
Attempt to convert dates to Pandas datetime values. Note that
|
some rarely used SAS date formats may be unsupported.
|
blank_missing : bool, defaults to True
|
Convert empty strings to missing values (SAS uses blanks to
|
indicate missing character variables).
|
chunksize : int, defaults to None
|
Return SAS7BDATReader object for iterations, returns chunks
|
with given number of lines.
|
encoding : str, 'infer', defaults to None
|
String encoding acc. to Python standard encodings,
|
encoding='infer' tries to detect the encoding from the file header,
|
encoding=None will leave the data in binary format.
|
convert_text : bool, defaults to True
|
If False, text variables are left as raw bytes.
|
convert_header_text : bool, defaults to True
|
If False, header text, including column names, are left as raw
|
bytes.
|
r/ Ú _int_lengthz bytes | NoneÚ _cached_pageNTÚinferz FilePath | ReadBuffer[bytes]Úboolz
|
int | Nonez
|
str | Noner r8 ) Ú path_or_bufÚ convert_datesÚ blank_missingÚ chunksizeÚencodingÚ convert_textÚconvert_header_textÚ compressionr' c
|
|
C sú || _ || _|| _|| _|| _|| _|| _d| _d| _g | _ g | _
|
g | _ g | _ g | _ d | _g | _g | _g | _d| _d| _d| _t|dd| d| _| jj| _| j| j| j| j| j | j | j | j d g | _ z| !¡ | "¡ W n t#k
|
rô | $¡ Y nX d S )Nzlatin-1ó r ÚrbF)Zis_textrL )%ÚindexrF rG rH rI rJ rK Údefault_encodingrL Úcolumn_names_rawÚ column_namesÚcolumn_formatsÚcolumnsÚ%_current_page_data_subheader_pointersrB Ú_column_data_lengthsÚ_column_data_offsetsÚ _column_typesÚ_current_row_in_file_indexZ_current_row_on_page_indexr ÚhandlesÚhandleÚ _path_or_bufÚ_process_rowsize_subheaderÚ _process_columnsize_subheaderÚ_process_subheader_countsÚ _process_columntext_subheaderÚ _process_columnname_subheaderÚ#_process_columnattributes_subheaderÚ_process_format_subheaderÚ _process_columnlist_subheaderÚ_subheader_processorsÚ_get_propertiesÚ_parse_metadataÚ ExceptionÚclose)
|
r: rE rO rF rG rH rI rJ rK rL r# r# r$ r; ¡ sX ÿ
|
÷ zSAS7BDATReader.__init__z
|
np.ndarray)r' c C s t j| jt jdS )z5Return a numpy int64 array of the column data lengths©Údtype)ÚnpÚasarrayrV Úint64©r: r# r# r$ Úcolumn_data_lengthsà s z"SAS7BDATReader.column_data_lengthsc C s t j| jt jdS )z0Return a numpy int64 array of the column offsetsrj )rl rm rW rn ro r# r# r$ Úcolumn_data_offsetsä s z"SAS7BDATReader.column_data_offsetsc C s t j| jt d¡dS )zj
|
Returns a numpy character array of the column types:
|
s (string) or d (double)
|
ZS1rj )rl rm rX rk ro r# r# r$ Ú column_typesè s zSAS7BDATReader.column_typesc C s | j ¡ d S r9 )rZ ri ro r# r# r$ ri ï s zSAS7BDATReader.closec C s
|
| j d¡ | j d¡| _| jdttj
tjkr<td| tj tj
|
¡}|tj krtd| _ d| _ tj| _tj| _n d| _ tj| _tj| _d| _ | tjtj¡}|tjkr²tj}nd}| tjtj¡}|dkrâd | _tj d
|
k| _ nd | _tj d k| _ | tj tj ¡d }|tj kr6tj | | _!| j"d krD| j!| _"nd| d| _!t#ddd}| $tj%| tj&¡}|t'j(|dd | _)| $tj*| tj+¡}|t'j(|dd | _,| -tj.| tj/¡| _0| j | j0d ¡}| j|7 _t| j| j0krðtd| -tj1| tj2¡| _3d S )Nr i z'magic number mismatch (not a SAS file?)Té Fé ó ú<Úbigú>ÚlittlerC zunknown (code=ú)r r r r) z*The SAS7BDAT file appears to be truncated.)4r\ ÚseekÚreadrB ÚlenÚconstÚmagicr" Ú _read_bytesZalign_1_offsetZalign_1_lengthZu64_byte_checker_valueÚU64rA Zpage_bit_offset_x64Ú_page_bit_offsetZ subheader_pointer_length_x64Ú_subheader_pointer_lengthZpage_bit_offset_x86Z subheader_pointer_length_x86Zalign_2_offsetZalign_2_lengthZalign_1_checker_valueZ align_2_valueZendianness_offsetZendianness_lengthÚ
|
byte_orderÚsysÚ byteorderÚ need_byteswapZencoding_offsetZencoding_lengthZencoding_namesZinferred_encodingrI r Ú _read_floatZdate_created_offsetZdate_created_lengthr! Z to_timedeltaZ date_createdZdate_modified_offsetZdate_modified_lengthZ date_modifiedÚ
|
_read_uintZheader_size_offsetZheader_size_lengthÚ header_lengthZpage_size_offsetZpage_size_lengthÚ _page_length)r: ÚbufZalign1ÚepochÚxr# r# r$ rf ò sj
|
|
|
|
ÿ ÿ ÿ ÿz SAS7BDATReader._get_propertiesr c C s( | j | jp dd}|jr$| ¡ t|S )Nr )Únrows)r| rH Úemptyri Ú StopIteration)r: Údar# r# r$ Ú__next__8 s
|
zSAS7BDATReader.__next__)ÚoffsetÚwidthc C sR | j d k st|dkr&t| j || jS |dkr>t| j || jS | ¡ tdd S )Nrt rs zinvalid float width)rB ÚAssertionErrorr r r ri r" ©r: r r r# r# r$ r @ s ÿ ÿzSAS7BDATReader._read_float)r r r' c C s | j d k st|dkr&| |d¡d S |dkr>t| j || jS |dkrVt| j || jS |dkrnt| j || jS | ¡ tdd S )Nr r é rt rs zinvalid int width) rB r r r r r r ri r" r r# r# r$ r O s. ÿ ÿ ÿzSAS7BDATReader._read_uint)r r7 c C sB | j d k st|| t| j kr0| ¡ td| j |||
S )Nz The cached page is too small.)rB r r} ri r" ©r: r r7 r# r# r$ r c s
|
zSAS7BDATReader._read_bytesr1 )r r7 r' c C s | | ||¡ d¡¡S )Nó )Ú_convert_header_textr Úrstripr r# r# r$ Ú _read_and_convert_header_textj s ÿz,SAS7BDATReader._read_and_convert_header_textc C sN d}|sJ| j | j¡| _t| jdkr(qJt| j| jkr@td| ¡ }qd S )NFr z2Failed to read a meta data page from the SAS file.)r\ r| r rB r} r" Ú_process_page_meta)r: Údoner# r# r$ rg o s z SAS7BDATReader._parse_metadatac C sZ | ¡ tjtjtjg }| j|kr,| ¡ | jtjk}| jtjk}t|pV|pV| j g kS r9 )
|
Ú_read_page_headerr~ Úpage_meta_typesZ page_amd_typeÚ page_mix_typeÚ_current_page_typeÚ_process_page_metadataÚpage_data_typerD rU )r: ÚptZ is_data_pageZ is_mix_pager# r# r$ r y s
|
ÿýz!SAS7BDATReader._process_page_metac C s^ | j }tj| }| |tj¡tj@ | _tj| }| |tj¡| _ tj
|
| }| |tj ¡| _ d S r9 ) r r~ Zpage_type_offsetr Zpage_type_lengthZpage_type_mask2r£ Zblock_count_offsetZblock_count_lengthZ_current_page_block_countZsubheader_count_offsetZsubheader_count_lengthÚ _current_page_subheaders_count)r: Ú
|
bit_offsetZtxr# r# r$ r s
|
ÿ
|
|
ÿz SAS7BDATReader._read_page_headerc C s | j }t| jD ]}tj| }|| j| }| || j¡}|| j7 }| || j¡}|| j7 }| |d¡}|d7 }| |d¡}|dks|tjkrq| || j¡} t
|
| }
|
| j |
|
} | d kr |tj dfk} |tj k} | jrô| rô| rô| j ||f¡ n| ¡ td| q| || qd S )Nr r z Unknown subheader signature )r Úranger§ r~ Zsubheader_pointers_offsetr r rA Ztruncated_subheader_idr r re Zcompressed_subheader_idZcompressed_subheader_typerL rU Úappendri r" )r: r¨ Úir Z total_offsetZsubheader_offsetZsubheader_lengthZsubheader_compressionZsubheader_typeZsubheader_signatureZsubheader_indexZsubheader_processorÚf1Úf2r# r# r$ r¤ s>
|
|
|
ÿþ
|
|
|
ÿÿz%SAS7BDATReader._process_page_metadatac C sÒ | j }|}|}| jr&|d7 }|d7 }n|d7 }|d7 }| |tj| |¡| _| |tj| |¡| _| |tj| |¡| _ | |tj
|
| |¡| _ tj | }| || |¡| _ | |d¡| _| |d¡| _d S )Niª i ib iz r )rA r r r~ Z row_length_offset_multiplierZ
|
row_lengthZrow_count_offset_multiplierÚ row_countZcol_count_p1_multiplierÚ col_count_p1Zcol_count_p2_multiplierÚ col_count_p2Z'row_count_on_mix_page_offset_multiplierZ_mix_page_row_countÚ_lcsÚ_lcp)r: r r7 Úint_lenZ
|
lcs_offsetZ
|
lcp_offsetZmxr# r# r$ r] ¾ s8
|
þ þ ÿ ÿ
|
z)SAS7BDATReader._process_rowsize_subheaderc C sT | j }||7 }| ||¡| _| j| j | jkrPtd| j d| j d| j d d S )Nz Warning: column count mismatch (z + z != z)
|
)rA r Ú column_countr¯ r° Úprint)r: r r7 r³ r# r# r$ r^ Ü s ÿz,SAS7BDATReader._process_columnsize_subheaderc C s d S r9 r# r r# r# r$ r_ ç s z(SAS7BDATReader._process_subheader_countsc C s || j 7 }| |tj¡}| ||¡}|d|
d¡}| j |¡ t| jdkrd}tj D ]}||kr\|}q\|| _
|
|| j 8 }|d }| j r|d7 }| || j ¡}| d¡}|dkrðd| _ |d }| j rÐ|d7 }| || j ¡}|d| j
| _n|tjkr4|d }| j r|d7 }| || j ¡}|d| j
| _nH| j dkr|d| _ |d }| j r^|d7 }| || j ¡}|d| j
| _t| d
|
r| | j¡| _d S ) Nr r r rM é rt ó é é( Ú creator_proc)rA r r~ Ztext_block_size_lengthr r rQ rª r} Zcompression_literalsrL r r² r± rº Zrle_compressionÚhasattrr ) r: r r7 Ztext_block_sizer Z cname_rawZcompression_literalZclZoffset1r# r# r$ r` ê sN
|
|
|
|
z,SAS7BDATReader._process_columntext_subheaderc C sÒ | j }||7 }|d| d d }t|D ]¢}|tj|d tj }|tj|d tj }|tj|d tj }| |tj¡} | |tj ¡}
|
| |tj
|
¡} | j | } | |
|
|
|
|
} | j | | ¡¡ q*d S )Nr é rs r )rA r© r~ Zcolumn_name_pointer_lengthZ!column_name_text_subheader_offsetZcolumn_name_offset_offsetZcolumn_name_length_offsetr Z!column_name_text_subheader_lengthZcolumn_name_offset_lengthZcolumn_name_length_lengthrQ rR rª r )r: r r7 r³ Zcolumn_name_pointers_countr« Ztext_subheaderZcol_name_offsetZcol_name_lengthÚidxÚ
|
col_offsetZcol_lenZname_rawÚcnamer# r# r$ ra sD ÿþÿ ÿþÿ ÿþÿ ÿ ÿ
|
z,SAS7BDATReader._process_columnname_subheaderc
|
C sÞ | j }|d| d |d }t|D ]²}|| tj ||d }|d| tj ||d }|d| tj ||d }| ||¡} | j | ¡ | |tj ¡} | j
|
| ¡ | |tj ¡} | j | dkrÒdnd¡ q&d S )Nr r¼ rs r ó dó s) rA r© r~ Zcolumn_data_offset_offsetZcolumn_data_length_offsetZcolumn_type_offsetr rW rª Zcolumn_data_length_lengthrV Zcolumn_type_lengthrX )
|
r: r r7 r³ Z column_attributes_vectors_countr« Zcol_data_offsetZ col_data_lenZ col_typesr r# r# r$ rb 8 s* ÿÿþ
|
ýÿ ÿ z2SAS7BDATReader._process_columnattributes_subheaderc C s d S r9 r# r r# r# r$ rd R s z,SAS7BDATReader._process_columnlist_subheaderc C sx | j }|tj d| }|tj d| }|tj d| }|tj d| }|tj d| }|tj d| } | |tj ¡}
|
t
|
|
|
t | j d } | |tj ¡} | |tj¡} | |tj¡}t
|
|t | j d }| |tj¡}| | tj¡}| j | }| ||||
¡}| j | }| || | |
¡}t | j}t|| j| ||| j| | j| }| j |¡ | j |¡ d S )Né r )rA r~ Z)column_format_text_subheader_index_offsetZcolumn_format_offset_offsetZcolumn_format_length_offsetZ(column_label_text_subheader_index_offsetZcolumn_label_offset_offsetZcolumn_label_length_offsetr Z)column_format_text_subheader_index_lengthÚminr} rQ Zcolumn_format_offset_lengthZcolumn_format_length_lengthZ(column_label_text_subheader_index_lengthZcolumn_label_offset_lengthZcolumn_label_length_lengthr rT r. rR rX rV rS rª )r: r r7 r³ Ztext_subheader_formatZcol_format_offsetZcol_format_lenZtext_subheader_labelZcol_label_offsetZ col_label_lenr Z
|
format_idxZ format_startZ
|
format_lenZ label_idxZ label_startZ label_lenZ label_namesZ column_labelZ format_namesZ column_formatZcurrent_column_numberÚcolr# r# r$ rc V s` ÿÿ ÿ ÿ ÿ ÿ
|
ÿ
|
ÿ
|
ú z(SAS7BDATReader._process_format_subheader)r r' c C sò |d kr| j d k r| j }n|d kr(| j}t| jdkrF| ¡ td|dkr`| j| jkr`t S t|| j| j }| j d¡}| j d¡}t
|
j ||ft d| _ t
|
j|d| ft
|
jd| _d| _t| }| |¡ | ¡ }| jd k rî| | j¡}|S )Nr z No columns to parse from filerÀ rÁ rj rs )rH r® r} rX ri r
|
rY r rà Úcountrl r ÚobjectÚ _string_chunkÚzerosZuint8Ú _byte_chunkÚ_current_row_in_chunk_indexr r| Ú_chunk_to_dataframerO Z set_index)r: r ZndÚnsÚpÚrsltr# r# r$ r| s*
|
|
zSAS7BDATReader.readc C s¦ g | _ | j | j¡| _t| jdkr(dS t| j| jkrf| ¡ dt| jdd| jdd}t|| ¡ | j t
|
j kr| ¡ | j t
|
j t
|
j t
|
jg kr¢| ¡ S dS )Nr Tz-failed to read complete page from file (read r z of z bytes)F)rU r\ r| r rB r} ri r" r r£ r~ r¡ r¤ r¥ r¢ Ú_read_next_page)r: Úmsgr# r# r$ rÏ ¬ s$ ÿ þz SAS7BDATReader._read_next_pagec C st | j }| j}t|| |}i }d\}}t| jD ]*}| j| }| j| dkrÚ| j|d d
f j| jd d} t j
|
| t j |d||< | j rÐ| j| tjkr®t|| d||< n"| j| tjkrÐt|| d||< |d7 }q0| j| dkr<t j
|
| j|d d
f |d ||< | jr2| jd k r2| || j¡||< |d7 }q0| ¡ td
|
t| j| q0t|| j|d d }
|
|
|
S ) N)r r rÀ r rj )rk rO r r rÁ )rO zunknown column type F)rT rO Úcopy) rÊ rY r© r´ rR rX rÉ Úviewr r! r, rl Úfloat64rF rS r~ Zsas_date_formatsr- Zsas_datetime_formatsrÇ rJ rI Ú_decode_stringr ri r" Úreprr ) r: ÚnÚmÚixrÎ ZjsZjbÚjr2 Zcol_arrZdfr# r# r$ rË Å s2
|
|
|
z"SAS7BDATReader._chunk_to_dataframec C s | | jp| j¡S r9 )ÚdecoderI rP ©r: Úbr# r# r$ rÔ ä s z SAS7BDATReader._decode_stringr5 )rÜ r' c C s | j r| |¡S |S d S r9 )rK rÔ rÛ r# r# r$ r ç s
|
z#SAS7BDATReader._convert_header_text)NTTNNTTrC )N)!r< r= r> Ú__doc__r? r; rp rq rr ri rf r r r r r rg r r r¤ r] r^ r_ r` ra rb rd rc r| rÏ rË rÔ r r# r# r# r$ r@ sN
|
ö ?F
|
+ -!7 r@ ).rÝ Ú
|
__future__r Ú collectionsr r r r
Útypingr Únumpyrl Zpandas._typingr r r Z pandas.errorsr
|
r Zpandasr! r r Zpandas.io.commonr Zpandas.io.sas._byteswapr r r r r Zpandas.io.sas._sasr r Zpandas.io.sas.sas_constantsÚioZsasZ sas_constantsr~ Zpandas.io.sas.sasreaderr r% r- r. ÚIteratorr@ r# r# r# r$ Ú<module> s&
|
