U
    vIdo                     @  s  d dl mZ d dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlZd dlZd dlZd d	lmZ d dlZd d
lmZmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z, d dl-m.Z. d dl"m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6 ej7Z8ej9Z:ej;Z<e1=e<e2dde1=e:e2dde1=e8e2dddd Z>e1=e$e2dde1=e#e2dddd Z?dd Z@eAdeAdddddddddddddd d!d"ZBdeAdeAdfd#d$d%ZCe1De$ee2d&d'e1De:ee2d&d'ei fd(d)ZEe1=e:e2d&d'e1=e:e2d*d'e1=e<e2d&d'e1=e<e2d*d'e1=e$e2d&d'e1=e$e2d*d'd+d, ZFe1De:ee2d*d'e1De$ee2d*d'ei fd-d.ZGe1=e:e2d/d'e1=e$e2d/d'd0d1 ZHe1De:eIe2d/d'e1De$eIe2d/d'ei fd2d3ZJe1De:eKe2d4d5e1De$eKe2d4d5ei fd6d7ZLe1De:e!jMe2d4d5e1De:ejNe2d4d5e1De:ej7e2d4d5e1De:ejOjPe2d4d5e1De$e!jMe2d4d5e1De$ejNe2d4d5e1De$ej7e2d4d5e1De$ejOjPe2d4d5ei fd8d9ZQe1De$e%e2d4d5e1De:e%e2d4d5ei fd:d;ZRe1=e8e2d4d5e1=e#e2d4d5e1=e#e2d<d5d=d> ZSe1Te8e2d4d5e1Te#e2d<d5deAddd#d?d@ZUe1Te#e2d4d5deAddd#dAdBZVe1=e8e2d<d5dCdD ZWe1Te8e2d<d5deAdfdEdFZXe1De:e!jMdGfe2d<d5e1De:e!jMdHfe2d<d5e1De:ejNdGfe2d<d5e1De:ejNdHfe2d<d5ei fdIdJZYe1De$e!jMdGfe2d<d5e1De$e!jMdHfe2d<d5e1De$ejNdGfe2d<d5e1De$ejNdHfe2d<d5ei fdKdLZZdMdMdNdOdPZ[e1=e8e2dQd5e1=e#e2dQd5dRdS Z\e1De:ejNdTfe2dQd5e1De:ej]e2dQd5ei fdUdVZ^e1De$ejNdTfe2dQd5e1De$ej]e2dQd5ei fdWdXZ_ei fdYdZd[d\Z`ee`d]dZZaee`d^dZZbe1De:ejce2d_d'ea e1De:e!jde2d_d'ea e1De:ejee2d`d'eb e1De:e!jfe2d`d'eb e1De$ejce2d_d'ea e1De$e!jde2d_d'ea e1De$ejee2d`d'eb e1De$e!jfe2d`d'eb e1De:ee2dd'e1De$ee2dd'ei fdadbZge1=e:e2d`d'e1=e:e2d_d'e1=e$e2d`d'e1=e$e2d_d'dcdd Zhe1Te:e2d`d'e1Te:e2d_d'e1Te$e2d`d'e1Te$e2d_d'deAdeAdfd#dedfZie1De:e/e2dgd'e1De$e/e2dgd'e1De:e!jje2dgd'e1De$e!jje2dgd'ei fdhdiZke1=e:e2dgd'e1=e$e2dgd'djdk Zle1De:e!jme2dld5e1De:ejne2dld5e1De$e!jme2dld5e1De$ejne2dld5ei fdmdnZoe1=e:e2dld5e1=e$e2dld5dodp Zpe1Te:e2dld5e1Te$e2dld5deAddeAddfd#dqdrZqe1=e:e2dld'e1=e$e2dld'dsdt ZrdudvdwdxdyZse1Te:e2dld'e1Te$e2dld'deAdeAdfd#dzd{Zte1De:ejue2d|d5e1De$ejue2d|d5ei fd}d~Zve1=e:e2d|d5e1=e$e2d|d5dd Zwe1Te:e2d|d5e1Te$e2d|d5deAdfd#ddZxe1De:ejyjze2dd'e1De$ejyjze2dd'e1De:ejyj{e2dd'e1De$ejyj{e2dd'ei fddZ|e1=e:e2dd'e1=e$e2dd'dd Z}e1=e:e2dd'e1=e$e2dd'dd Z~e1=e8e2dd5e1=e#e2dd5dd Zei fddZei fddZeejejejejejeejejejejefej ej D ]6Ze1De:ee2dd5e e1De$ee2dd5e qe1De$ee2dd5e e1De$eje2dd5e e1=e8e2dd5dd Ze1=e#e2dd5dd Ze1=e8e2dd5e e1=e#e2dd5e e1De:eje2dd5e1De:ee2dd5dd ZdS )    )annotations)PathLike)Mapping)partial)UnionLiteral)MappingProxyType)warnN)sparse)AnnDataRaw)_normalize_indices)intersect_keysSparseDataset)views)	ZarrArray	ZarrGroup	DaskArray
_read_attr_from_fixed_length_strings_decode_structured_array)report_write_key_on_error	check_keyH5PY_V3)OldFormatWarning)AwkArray   )	_REGISTRYIOSpecget_spec	read_elemread_elem_partial
write_elem c                   st   ddl m} td| j dtdd t| trZd| jkrDt| 	 S  fdd	| 
 D S t| tjrp|| S d S )
Nr   )h5ad	Element '(' was written without encoding metadata.   
stacklevelh5sparse_formatc                   s   i | ]\}}|  |qS  r!   .0kv_readerr,   R/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/anndata/_io/specs/methods.py
<dictcomp>^   s      zread_basic.<locals>.<dictcomp>)anndata._ior%   r	   namer   
isinstancer   attrsr   	to_memoryitemsh5pyDatasetread_dataset)elemr3   r%   r,   r2   r4   
read_basicN   s    

r@   c                   sr   ddl m} td| j dtdd t| trZd| jkrDt| 	 S  fdd	| 
 D S t| trn|| S d S )
Nr   )zarrr&   r'   r(   r)   r+   c                   s   i | ]\}}|  |qS r,   r-   r.   r2   r,   r4   r5   r   s      z#read_basic_zarr.<locals>.<dictcomp>)r6   rA   r	   r7   r   r8   r   r9   r   r:   r;   r   r>   )r?   r3   rA   r,   r2   r4   read_basic_zarrc   s    


rB   c                 C  sH   | d }|t |jd }t|}| d }|t |jd }t|}||fS )Nobs_indexvar)r   r9   r!   )groupZ	obs_groupZobs_idx_elemobs_idxZ	var_groupZvar_idx_elemvar_idxr,   r,   r4   read_indices   s    rI   T)rG   rH   XrC   rE   obsmvarmobspvarplayersunsr   z
ad.AnnData)pthreturnc             	   C  s  i }t | dr}t||fft| \}}t|d ||td fd|d< t|d ||td fd|d< |rt|d ||fd|d< n"tt|d t|d f|d< d|krt	|d ||td fd|d< d|krt	|d ||td fd|d< d	|krt	|d	 |||fd|d	< d
|kr>t	|d
 |	||fd|d
< d|krbt	|d |
||fd|d< d|krt	|d |d|d< W 5 Q R X t
jf |S )NrrC   r;   indicesrE   rJ   rU   rK   rL   rM   rN   rO   rP   r;   )r<   Filer   rI   r"   slicer
   
csr_matrixlen_read_partialadr   )rQ   rG   rH   rJ   rC   rE   rK   rL   rM   rN   rO   rP   resultfr,   r,   r4   read_partial   sb      

  

"  

  


  

  

  

r`   rT   c                C  sp   | d krd S |d kr t | f}nt | |f}i }|D ]6}t|trP||d }nd }t| | ||d||< q4|S NrT   )r   r8   r   getr"   )rF   r;   rU   keysr^   r0   Z
next_itemsr,   r,   r4   r\      s    
r\   anndata0.1.0c                 C  s   |  |}|j|d|j|d |j|d|j|d |j|d|j|d |j|dt|j|d |j|dt|j|d |j|dt|j|d |j|dt|j	|d |j|d	t|j
|d |j|d
t|j|d |j|d|j|d d S )NrJ   dataset_kwargsrC   rE   rK   rL   rM   rN   rO   rP   raw)Zrequire_groupr#   rJ   rC   rE   dictrK   rL   rM   rN   rO   rP   rh   )r_   r0   Zadata_writerrg   gr,   r,   r4   write_anndata   s    
rl   rh   c                 C  s2   i }dD ]}|| kr| | | ||< qtf |S )N)
rJ   rC   rE   rK   rL   rM   rN   rO   rP   rh   )r!   r   )r?   r3   dr0   r,   r,   r4   read_anndata   s
    rn   c                 C  sN   |  |}|j|d|j|d |j|d|j|d |j|dt|j|d d S )NrJ   rf   rE   rL   )create_groupr#   rJ   rE   ri   rL   )r_   r0   rh   rj   rg   rk   r,   r,   r4   	write_raw   s    
rp   ri   c                   s    fdd|   D S )Nc                   s   i | ]\}}|  |qS r,   r-   r.   r2   r,   r4   r5     s      z read_mapping.<locals>.<dictcomp>rW   r?   r3   r,   r2   r4   read_mapping  s    rr   c                 C  s2   |  |}| D ]\}}|j||||d qd S Nrf   )ro   r;   r#   )r_   r0   r1   rj   rg   rk   Zsub_kZsub_vr,   r,   r4   write_mapping  s    
rt   arrayz0.2.0c                 C  s   |j | |t||d d S rs   )r#   npru   r_   r0   r?   rj   rg   r,   r,   r4   
write_list!  s    rx   c                 C  s   | j |fd|i| dS )z8Write methods which underlying library handles natively.dataN)create_datasetrw   r,   r,   r4   write_basic)  s    
r{   c                 C  s8   dd l m} | j|f|j|jd|}||| d S )Nr   )shapedtype)Z
dask.arrayru   Zrequire_datasetr|   r}   store)r_   r0   r?   rj   rg   dark   r,   r,   r4   write_basic_dask6  s    r   zstring-arrayc                 C  s   | d S Nr,   r,   rq   r,   r,   r4   
read_array?  s    r   c                C  s   | | S Nr,   r?   r;   rU   r,   r,   r4   read_array_partialF  s    r   c                C  s
   | j | S r   )Zoindexr   r,   r,   r4   read_zarr_array_partialL  s    r   c                 C  s   t |  |dS )Nr2   )r   asstr)rm   r3   r,   r,   r4   read_string_arrayR  s    r   c                 C  s   t |  ||dS ra   )r   r   )rm   r;   rU   r,   r,   r4   read_string_array_partialW  s    r   UOc                 C  s.   t jtd}| j|f|||d| dS )z8Write methods which underlying library handles nativley.Zvlen)ry   r}   N)r<   special_dtypestrrz   astype)r_   r0   r?   rj   rg   Z	str_dtyper,   r,   r4   write_vlen_string_array\  s    
r   c                 C  s<   dd l }| j|f|jt| d| || | d d < d S )Nr   )r|   r}   Zobject_codec)	numcodecsrz   r|   objectZVLenUTF8)r_   r0   r?   rj   rg   r   r,   r,   r4   write_vlen_string_array_zarrj  s    r   z
np.ndarray)valuerR   c                 C  sX   g }| j j D ]<\}\}}|jdkr>||tjtdf q|||f q| |S )z6This corrects compound dtypes to work with hdf5 files.)r   r   r   )	r}   fieldsr;   kindappendr<   r   r   r   )r   Z	new_dtypeZdt_nameZdt_type_r,   r,   r4   _to_hdf5_vlen_strings  s    
r   z	rec-arrayc                 C  s*   | d }|j }t|}tr&t||d}|S )Nr,   r}   )r}   r   r   r   )rm   r3   r   r}   r,   r,   r4   read_recarray  s    r   Vc                 C  s   | j |fdt|i| d S Nry   )rz   r   rw   r,   r,   r4   write_recarray  s    r   c                 C  s*   ddl m} | j|fd||i| d S )Nr   )_to_fixed_length_stringsry   )anndata.compatr   rz   )r_   r0   r?   rj   rg   r   r,   r,   r4   write_recarray_zarr  s    r   zLiteral[('csr', 'csc')])fmtc                 C  s   |  |}|j|jd< t| tr:d|kr:tf ddi|}|jdd|ji| |jdd|ji| |jd	d|j	i| d S )
Nr|   Zmaxshaper   ry   rU   indptr)ry   )rU   )r   )
ro   r|   r9   r8   H5Groupri   rz   ry   rU   r   )r_   keyr   rj   r   rg   rk   r,   r,   r4   write_sparse_compressed  s    
r   ZcsrZcscrZ   
csc_matrixc                 C  sB   t | || ||j|d |j d| | jd< d| | jd< d S )N)r   rg   Z_matrixzencoding-typere   zencoding-version)r   Z	to_backed
format_strr9   rw   r,   r,   r4   write_sparse_dataset  s    	r   c                 C  s   t |  S r   )r   r:   rq   r,   r,   r4   read_sparse  s    r   c                C  s   t | | S r   r   r   r,   r,   r4   read_sparse_partial  s    r   zawkward-arrayc           
      C  sl   ddl m} | |}|||\}}}	||jd< | |jd< |	 D ]\}}|j||||d qLd S )Nr   awkwardlengthformrf   )	r   r   ro   Z
to_buffersZ	to_packedr9   to_jsonr;   r#   )
r_   r0   r1   rj   rg   akrF   r   r   	containerr,   r,   r4   write_awkward  s    	

r   c                   sJ   ddl m} tjd}tjd} fdd D }||||S )Nr   r   r   r   c                   s   i | ]}|  | qS r,   r-   r/   r0   r3   r?   r,   r4   r5     s      z read_awkward.<locals>.<dictcomp>)r   r   r   r9   rc   Zfrom_buffers)r?   r3   r   r   r   r   r,   r   r4   read_awkward  s
    r   Z	dataframec                 C  s   dD ]}||j krt|dq| |}dd |j D }||jd< |jjd k	r\|jj}nd}t||jd< |j|||jj|d |	 D ]\}	}
|j||	|
j|d qd S )N)rD   z* is a reserved name for dataframe columns.c                 S  s   g | ]}t |qS r,   )r   )r/   cr,   r,   r4   
<listcomp>2  s     z#write_dataframe.<locals>.<listcomp>column-orderrD   rf   )
columns
ValueErrorro   r9   indexr7   r   r#   _valuesr;   )r_   r   dfrj   rg   reservedrF   Z	col_namesZ
index_nameZcolnameZseriesr,   r,   r4   write_dataframe(  s.    



      r   c                   sf   t tjd}tjd}tj fdd|D  | t|rJ|nd d}|dkrb||j_|S )Nr   rD   c                   s   i | ]}|  | qS r,   r-   r   r   r,   r4   r5   N  s      z"read_dataframe.<locals>.<dictcomp>r   r   )	listr   r9   pd	DataFramer!   r[   r   r7   r?   r3   r   idx_keyr   r,   r   r4   read_dataframeH  s    r   c                  s   d k	r$fddt  jdD }ntt  jd}t  jd}tj fdd|D t | d dt|rt|nd d	}|dkr||j_|S )
Nc                   s   g | ]}| kr|qS r,   r,   )r/   colrW   r,   r4   r   ^  s     z*read_dataframe_partial.<locals>.<listcomp>r   rD   c                   s"   i | ]}|t  | d  dqS )r   rV   )r"   r   )r?   rU   r,   r4   r5   e  s      z*read_dataframe_partial.<locals>.<dictcomp>r   rV   r   )	r   r9   r   r   r   r"   r[   r   r7   )r?   r;   rU   r   r   r   r,   )r?   rU   r;   r4   read_dataframe_partialX  s    

r   c                   s^   t  jd}t  jd}tj fdd|D t | t|rB|nd d}|dkrZ||j_|S )Nr   rD   c                   s   i | ]}|t  | qS r,   )read_seriesr   r?   r,   r4   r5   w  s      z(read_dataframe_0_1_0.<locals>.<dictcomp>r   )r   r9   r   r   r   r[   r   r7   r   r,   r   r4   read_dataframe_0_1_0q  s    
r   zh5py.Datasetz!Union[np.ndarray, pd.Categorical])datasetrR   c                 C  s   d| j krt| tr<dd l}| j| j}|| j| }n| j	}|t
| j d }t|}tt
|j dd}tjjt| ||dS t| S d S )N
categoriesr   orderedF)r   )r9   r8   r   rA   r7   rstripbasenameopenr~   parentr   r!   boolr   Categorical
from_codes)r   rA   parent_namer   Zcategories_dsetr   r   r,   r,   r4   r     s    

  r   c                C  s0   |d krt d }nt|}t| | j|d  S )Nr   )rY   r   r!   Zilocr   r,   r,   r4   read_partial_dataframe_0_1_0  s    
r   Zcategoricalc                 C  sH   |  |}t|j|jd< |j|d|j|d |j|d|jj|d d S )Nr   codesrf   r   )ro   r   r   r9   r#   r   r   r   r_   r0   r1   rj   rg   rk   r,   r,   r4   write_categorical  s    
   r   c                 C  s2   t jj|| d || d tt| jddS )Nr   r   r   r   r   r   )r   r   r   r!   r   r   r9   rq   r,   r,   r4   read_categorical  s
    r   c                C  s2   t jjt| d |dt| d tt| jddS )Nr   rV   r   r   r   )r   r   r   r"   r!   r   r   r9   r   r,   r,   r4   read_partial_categorical  s
    
r   znullable-integerznullable-booleanc                 C  s@   |  |}|jd k	r(|j|d|j|d |j|d|j|d d S )Nmaskrf   values)ro   Z_maskr#   _datar   r,   r,   r4   write_nullable_integer  s    

r   c                 C  sD   d| kr,t jj|| d || d dS t || d S d S Nr   r   )r   )r   arraysIntegerArrayr!   ru   rq   r,   r,   r4   read_nullable_integer  s     r   c                 C  sD   d| kr,t jj|| d || d dS t || d S d S r   )r   r   BooleanArrayr!   ru   rq   r,   r,   r4   read_nullable_boolean  s     r   znumeric-scalarc                 C  s   | d S r   r,   rq   r,   r,   r4   read_scalar  s    r   c                 C  s   | j |fdt|i|S r   )rz   rv   ru   r_   r   r   rj   rg   r,   r,   r4   write_scalar  s    r   c                 C  s@   |  }|dd  |dd  | j|fdt|i| d S )Ncompressioncompression_optsry   )copypoprz   rv   ru   r   r,   r,   r4   write_hdf5_scalar  s    r   stringc                 C  s   |   d S r   )r   rq   r,   r,   r4   read_hdf5_string  s    r   c                 C  s   t | d S r   )r   rq   r,   r,   r4   read_zarr_string  s    r   bytesc                 C  sL   |  }|dd  |dd  | j|fdtj|tjdddi| d S )Nr   r   ry   zutf-8)encodingr   )r   r   rz   rv   ru   r<   Zstring_dtype)r_   r0   r1   rj   rg   r,   r,   r4   write_string(  s    r   )
__future__r   osr   collections.abcr   	functoolsr   typingr   r   typesr   warningsr	   r<   numpyrv   Zpandasr   Zscipyr
   rd   r]   r   r   Zanndata._core.indexr   Zanndata._core.merger   Zanndata._core.sparse_datasetr   Zanndata._corer   r   r   r   r   r   r   r   Zanndata._io.utilsr   r   r   Zanndata._warningsr   r   registryr   r   r    r!   r"   r#   r=   ZH5ArrayGroupr   rX   ZH5FileZregister_readr@   rB   rI   rY   r`   r\   Zregister_writerl   rn   rp   rr   ri   rt   r   rx   Z	ArrayViewZndarraymaZMaskedArrayr{   r   r   Zregister_read_partialr   r   r   r   r   r   r   r   Zrecarrayr   r   r   Z	write_csrZ	write_cscrZ   ZSparseCSRViewr   ZSparseCSCViewr   r   r   ZAwkwardArrayViewr   r   ZDataFrameViewr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zbool_Zuint8Zuint16Zuint32Zuint64intZint8Zint16Zint32Zint64floatZfloating__subclasses__ZcomplexfloatingZnumeric_scalar_typer   Zstr_r   r   r   r,   r,   r,   r4   <module>   s,    	"!6 
        	$	      
        	
        

