U mdp@sBdZddlmZddlmZmZmZddlZddl Z ddl m Z ddl mZmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZeedZedZddZd(ddZddZGdddZ d)e e!ee"eedee!fe!ee#e"e"ee!e"eee"ee!ee dddZ$dd Z%d*e dd%d&d'Z&dS)+z1Rank genes according to differential expression. )floor)IterableUnionOptionalN)AnnData)issparsevstack)_utils)logging) _get_mean_var)Literal) _get_obs_rep)check_nonnegative_integers)logregt-testwilcoxont-test_overestim_var)benjamini-hochberg bonferronicCsX|jd}tj|td}t|| | d}t||ddd}|||}|S)Nr)Zdtype)shapenparangeintZ argpartitionZargsort)scoresZn_topZn_fromZreference_indices partitionZpartial_indicesglobal_indicesrX/home/sam/Atlas/atlas_env/lib/python3.8/site-packages/scanpy/tools/_rank_genes_groups.py _select_top_ns   r c #sd}|jd}t|r(ddddntjdddk oDdk }|rptt}fdd}n|jd}fd d}t||}td||D]:} t| ||} tj ||| | d } | } | | | fVqdS) NicSs t|SN)rtoarray)Ztplrrr'z_ranks..cSs|Sr")r#Xrrrr$(r%cSs|Sr"rr&rrrr$+r%cs$|||f|||ffSr"rr'leftright)mask mask_restmergerrr$1srcs|dd||fSr"rr()adaptrrr$6r%)data) rrrr count_nonzerorrangeminpd DataFrameZrank) r'r+r,ZCONST_MAX_SIZEn_genesZmaskedn_cellsZ get_chunkZ max_chunkr)r*Zdfranksr)r.r+r,r-r_ranks!s&     r8cCst|jd}|dkr*t|jddStj|dd}tj|dd|ddkd|jddfddd}t|t|jddddfd}tj|dd}tj|dd tj}d|d|j dd|d|S) Nrr r!?ZaxisrT) rfloat64rrepeatsortinsertwhererdiffZastypesum)r7sizeZarrtfidxZcntrrr _tiecorrectCs4&rFc@s@eZdZdddZddZd d Zd d Zd dZdddZdS) _RankGenesrestTNFc svdkr.jdddk r.fdd|_ntj|_t||\|_|_t |jt j | j ddj @}t|dkrtdd|} |dk r|rtd | j|} n|r̈jdk r̈j} | j} t| r| | |_| j|_d|_|d krt|j|kdd|_d|_d|_d|_d|_||_d|_d|_ d|_!j |"|j|_#j j |j#|f|_$dS) Nlog1pbasecst|tjddSNrIrJrexpm1logunsxadatarrr$^r%z%_RankGenes.__init__..cSs|dkS)Nr rrPrrrr$hr%rzPCould not calculate statistics for groups {} since they only contain one sample.z, z/Cannot specify `layer` and have `use_raw=True`.rH)%uns_keysrO expm1_funcrrMr Z select_groups groups_order groups_maskssetobsZ value_countslocindexlen ValueErrorformatjoinZlayersrawr'rZeliminate_zeros var_names ireferencer@meansvars means_rest vars_restcomp_ptsptspts_reststatsisin grouping_maskgrouping) selfrSgroupsgroupby referenceuse_rawlayerrgZinvalid_groups_selectedZ adata_compr'rrRr__init__RsT      z_RankGenes.__init__c Cs|jjd}|jjd}t||f|_t||f|_|jrLt||fnd|_|j dkrt||f|_ t||f|_ |jrt||fnd|_ n4|j|j }|j|}t |\|j|j <|j|j <~t|jrdd}ndd}t|jD]\}}|j|}|jr$|||jd|j|<|j dk r>||j kr>qt |\|j|<|j|<|j dkr|}|j|}t |\|j |<|j |<|jr|||jd|j |<~qdS)Nr!rcSs |jddSNrr:)getnnzr&rrrr$r%z)_RankGenes._basic_stats..cSstj|ddSru)rr0r&rrrr$r%)r'rrWrzerosrcrdrgrhrbrerfrir r enumerate) rnr5n_groupsr,ZX_restZ get_nonzerosimaskr+ZX_maskrrr _basic_statss@          z_RankGenes._basic_statsc csHddlm}|t|jD]"\}}|jdk r>||jkr>q|j|}|j|}t |}|jdk r|j|j}|j|j} t |j|j} n$|j |}|j |} |j j d|} |dkr| } n|dkr|} ntdtjdd.|j|t|||t| | dd \} } W5QRXd| t| <d | t| <|| | fVqdS) NrrjrrzMethod does not exist.ignore)invalidF)Zmean1Zstd1Znobs1Zmean2Zstd2Znobs2Z equal_varr!)scipyrjr{rxrWrbrcrdrr0rerfr'rr]ZerrstateZttest_ind_from_statssqrtisnan)rnmethodrj group_indexr+ mean_groupZ var_groupZns_group mean_restZvar_restZns_otherZns_restrpvalsrrrt_testsB          z_RankGenes.t_testc csddlm}||jjd}|jdk rlt|}|rFt|}nd}t|j D]\}}||jkrjqT|j |j}t |} t |} | dks| dkrt dt |j||D]B\} } } t| jd| ddf|| | <|rt| || | <qt|| | | | dd}|| | | dd|}d|t|<d|jjt|}|||fVqTnn|j jd}t||f}|jjd}|rt||f}t |jD]`\} } } t|j D]H\}}t| j|ddf||| | f<|rt| ||| | f<qĐqt|j D]\}}t |} |r>||}nd}t|| || |dd}||ddf| |dd|||ddf<d|t|<d|jjt||ddf}||||fVqdS) Nrr|r!zQFew observations in a group for normal approximation (<=25). Lower test accuracy.g(@g@r )rrjr{r'rrbrrwrxrWr0logghintr8rBZilocrFrr distributionsZnormZsfabs)rn tie_correctrjr5rTrr+r,Zn_activeZm_activer7r)r*Zstd_devrryr6rzZT_irrrrsn         $  $  $z_RankGenes.wilcoxonc ksddlm}|j|jjddf}t|jdkr8td|f|}|||j j j |j }t |jD]F\}}t|jdkr|d}n||}||dfVt|jdkrdqqddS)Nr)LogisticRegressionr!z7Cannot perform logistic regression on a single cluster.r )Zsklearn.linear_modelrr'rlvaluesr\rVr]fitrmcatcodesZcoef_rx) rnkwdsrr'ZclfZ scores_allZigroup_rrrrrLs    z_RankGenes.logregrcKs|dkr||}n(|dkr(||}n|dkr<|jf|}d|_|jjd}|D]\} } } t|j| } |dk r|rt | n| } t | |}d}n t d}d}|jdkrt j | |fg}t j|d|_|dk r|j||j| df<| ||j| df<| dk r~| ||j| df<|d krRd d lm}d| t| <|| d d d\}}}}n|dkrlt| |d}|||j| df<|jdk rR|j| }|jdkr|j| }n |j|j}||d||d}t|||j| df<qR|dkr|j|j_dS)N>rrrrr!namesr)columnsrrr) multipletestsg?Zfdr_bh)alpharrr9 pvals_adj& .>logfoldchanges)rrrrjr'rstrrVrrr slicer3Z MultiIndex from_tuplesr4raZstatsmodels.stats.multitestrrminimumrcrbrerUlog2r[)rnr corr_method n_genes_user rankby_absrrZgenerate_test_resultsr5rrr group_nameZ scores_sortrZ first_colrErrrrrZ foldchangesrrrcompute_statisticses`                   z_RankGenes.compute_statistics)rHTNF)rNFF) __name__ __module__ __qualname__rtr{rrrrrrrrrGQs D/2UrGallrHFr)rSrprrrorqr5rrh key_addedcopyrrrrsreturncKsF|dkr|jdk }n|dkr.|jdkr.td| dkrDtdd} d|krX|d }td}ddd d h}| |krtd |d d dh}| |krtd|d | r|n|}t||dkrd}n`t |t t frtdnHt |}t |dt r dd|D}|dkr.|t |kr.||g7}|dkrt||j|jjkrt|j|jj}td|d|d |dkrd}i|j|<t||| || | d|j|d<t|||||| |}t|jr| d krtd|}|dks||jjdkr |jjd}td|dtdtj|jdd |j| | ||| f||jdk rd!d|jD}tj |jj!|j"|d"|j|d#<|j#dk rtj |j#j!|j"|d"|j|d$<|j$j%&|j$_%d%d&d&d'd'd(}|j$j%j'dD](}|j$|j(d)||d*|j||<qtjd+|d,|d-| d.kr.d/nd0d1| rB|SdS)2a9 Rank genes for characterizing groups. Expects logarithmized data. Parameters ---------- adata Annotated data matrix. groupby The key of the observations grouping to consider. use_raw Use `raw` attribute of `adata` if present. layer Key from `adata.layers` whose value will be used to perform tests on. groups Subset of groups, e.g. [`'g1'`, `'g2'`, `'g3'`], to which comparison shall be restricted, or `'all'` (default), for all groups. reference If `'rest'`, compare each group to the union of the rest of the group. If a group identifier, compare with respect to this group. n_genes The number of genes that appear in the returned tables. Defaults to all genes. method The default method is `'t-test'`, `'t-test_overestim_var'` overestimates variance of each group, `'wilcoxon'` uses Wilcoxon rank-sum, `'logreg'` uses logistic regression. See [Ntranos18]_, `here `__ and `here `__, for why this is meaningful. corr_method p-value correction method. Used only for `'t-test'`, `'t-test_overestim_var'`, and `'wilcoxon'`. tie_correct Use tie correction for `'wilcoxon'` scores. Used only for `'wilcoxon'`. rankby_abs Rank genes by the absolute value of the score, not by the score. The returned scores are never the absolute values. pts Compute the fraction of cells expressing the genes. key_added The key in `adata.uns` information is saved to. **kwds Are passed to test methods. Currently this affects only parameters that are passed to :class:`sklearn.linear_model.LogisticRegression`. For instance, you can pass `penalty='l1'` to try to come up with a minimal set of genes that are good predictors (sparse solution meaning few non-zero fitted coefficients). Returns ------- **names** : structured `np.ndarray` (`.uns['rank_genes_groups']`) Structured array to be indexed by group id storing the gene names. Ordered according to scores. **scores** : structured `np.ndarray` (`.uns['rank_genes_groups']`) Structured array to be indexed by group id storing the z-score underlying the computation of a p-value for each gene for each group. Ordered according to scores. **logfoldchanges** : structured `np.ndarray` (`.uns['rank_genes_groups']`) Structured array to be indexed by group id storing the log2 fold change for each gene for each group. Ordered according to scores. Only provided if method is 't-test' like. Note: this is an approximation calculated from mean-log values. **pvals** : structured `np.ndarray` (`.uns['rank_genes_groups']`) p-values. **pvals_adj** : structured `np.ndarray` (`.uns['rank_genes_groups']`) Corrected p-values. **pts** : `pandas.DataFrame` (`.uns['rank_genes_groups']`) Fraction of cells expressing the genes for each group. **pts_rest** : `pandas.DataFrame` (`.uns['rank_genes_groups']`) Only if `reference` is set to `'rest'`. Fraction of cells from the union of the rest of each group expressing the genes. Notes ----- There are slight inconsistencies depending on whether sparse or dense data are passed. See `here `__. Examples -------- >>> import scanpy as sc >>> adata = sc.datasets.pbmc68k_reduced() >>> sc.tl.rank_genes_groups(adata, 'bulk_labels', method='wilcoxon') >>> # to visualize the results >>> sc.pl.rank_genes_groups(adata) NTz2Received `use_raw=True`, but `adata.raw` is empty.zNDefault of the method has been changed to 't-test' from 't-test_overestim_var'rZ only_positivez ranking genesrrrzMethod must be one of .rrz!Correction method must be one of rzSpecify a sequence of groupsrcSsg|] }t|qSrr).0nrrr 9sz%rank_genes_groups..rHz reference = z needs to be one of groupby = rank_genes_groups)rprqrrrrsrparamszyIt seems you use rank_genes_groups on the raw count data. Please logarithmize your data before calling rank_genes_groups.r!z consider z groups:z with sizes: r:cSsg|] }t|qSrr)rnamerrrres)r[rrhriOZfloat32r<)rrrrrF)r[Z column_dtypesz finishedzadded to `.uns[zz]` 'names', sorted np.recarray to be indexed by group ids 'scores', sorted np.recarray to be indexed by group ids >rrrz 'logfoldchanges', sorted np.recarray to be indexed by group ids 'pvals', sorted np.recarray to be indexed by group ids 'pvals_adj', sorted np.recarray to be indexed by group ids)timedeep))r`r]rwarningpopinforr Zsanitize_anndata isinstancerrlistrXrYr categoriestolistrOdictrGrr'rdebugrr0rWrrhrVr3r4rrarirjrZ swaplevelZlevels to_records)rSrprrrorqr5rrhrrrrrrsrstartZ avail_methodsZ avail_corrrVZcatsZtest_objrZ groups_namesZdtypescolrrrrsk               rcCs2t|r|jdd}ntj|dd}||jdSru)rrvrr0r)r'Z n_nonzerorrr _calc_fracsrrank_genes_groups_filtered?r!?)rSrc s|dkr d}|dkr&j|dd}|dkr@j|dd}j|dd|koj|dddkoj|dd|k} | odj|k} | od j|k} tj|d } tjt| j| j| jd } tjt| j| j| jd }| rtj|d}nRtjt| j| j| jd }d krZjd d dk rZfdd}ntj }t d|d|d|| jD]>}| |j }| r| s|rj dd|fjndd|fj}j||k}||}||}| rBj|d|j|j | jdd|f<j|d |j|j |jdd|f<n,t|| jdd|f<t||jdd|f<| st|d}t|d}t||d||d|jdd|f<q|r|}| | |k||k@||k@} j|j|<| jddj|d <dS)a Filters out genes based on log fold change and fraction of genes expressing the gene within and outside the `groupby` categories. See :func:`~scanpy.tl.rank_genes_groups`. Results are stored in `adata.uns[key_added]` (default: 'rank_genes_groups_filtered'). To preserve the original structure of adata.uns['rank_genes_groups'], filtered genes are set to `NaN`. Parameters ---------- adata key groupby use_raw key_added min_in_group_fraction min_fold_change max_out_group_fraction compare_abs If `True`, compare absolute values of log fold change with `min_fold_change`. Returns ------- Same output as :func:`scanpy.tl.rank_genes_groups` but with filtered genes names set to `nan` Examples -------- >>> import scanpy as sc >>> adata = sc.datasets.pbmc68k_reduced() >>> sc.tl.rank_genes_groups(adata, 'bulk_labels', method='wilcoxon') >>> sc.tl.filter_rank_genes_groups(adata, min_fold_change=3) >>> # visualize results >>> sc.pl.rank_genes_groups(adata, key='rank_genes_groups_filtered') >>> # visualize results using dotplot >>> sc.pl.rank_genes_groups_dotplot(adata, key='rank_genes_groups_filtered') NrrrprrrqrHrrir)rr[rIrJcst|tjddSrKrLrPrRrrr$r%z*filter_rank_genes_groups..z.Filtering genes using: min_in_group_fraction: z min_fold_change: z, max_out_group_fraction: rhrrF)r[)rOr3r4rrwrrr[rTrMrrrr`r'rYrZrZravelZmeanrrrr)rSkeyrprrrZmin_in_group_fractionZmin_fold_changeZmax_out_group_fractionZ compare_absZ same_paramsZ use_logfoldsZ use_fractionZ gene_namesZfraction_in_cluster_matrixZfraction_out_cluster_matrixZfold_change_matrixrUZclusterraZsub_XZin_groupZX_inZX_outZmean_in_clusterZmean_out_clusterrrRrfilter_rank_genes_groupss4   "   ,   r)NN) NrrHNFFNFNrFN)NNNrrr!rF)'__doc__mathrtypingrrrnumpyrZpandasr3ZanndatarZ scipy.sparserrrr r rZpreprocessing._simpler Z_compatr getrr_MethodZ _CorrMethodr r8rFrGrboolrrrrrrrrs~          "a d