
    Q3jx                       d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z% d d	l&m'Z' d d
l(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7m8Z8m9Z9 eeee gZ:ddZ;d Z<d Z=d Z>d Z?ej                  j                  dddg      d        ZBd ZCd ZDd ZEd ZFd ZGd ZHd ZId  ZJd! ZKd" ZLd# ZMej                  j                  d$ ej                  g d%      g d&f ej                  g d%      dfg d'g d(fg d'dfg      d)        ZOej                  j                  d$ ej                  g d*      g d&f ej                  g d+      g d,fg      d-        ZPej                  j                  d$ ej                  g d.      dfg d/dfg d.g d&fg d/g d(fg      d0        ZQej                  j                  d1g d2      d3        ZRd4 ZSej                  j                  d5d6 ej                  g d.      g d7fd6 ej                  g d/      g d8fd9 ej                  g d*      dfd: ej                  g d/      g d;fd< ej                  g d.      d d=gfd< ej                  g d/      d>d?gfd@ ej                  g d.      g dAfd@ ej                  g d/      g dBfdC ej                  g dD      g d(fdC ej                  g dB      g d(fdC ej                  g dA      g d&fg      ej                  j                  dEdFdGg      dH               ZTej                  j                  dIdJdKdFdLfdMdKdGdLfdNdFg dOfdPdFdQdRfdSi fg      dT        ZUdU ZVdV ZWdW ZXdX ZYdY ZZej                  j                  dZe:      d[        Z[ej                  j                  dZe:      d\        Z\ej                  j                  d]      ej                  j                  dZee g      ej                  j                  d^d_d`g      da                      Z^ej                  j                  dZe:      db        Z_ej                  j                  dddg      dc        Z`dd Zaej                  j                  dddg      de        Zbdf Zcdg Zddh Zedi Zfdj Zgdk Zhdl Ziej                  j                  dmg dng dod gd gfg dng dpd gd gfg dng dqd gd gfg dng drd gd gfg dsg dog dtg dufg dsg dpg dtg dufg dsg dqg dtg dufg dsg drg dtg dufg dug dodvgd gfg dug dpd gd gfg dug dqd gd gfg dug drd gd gfg dwg dog dxg dyfg dwg dpg dxg dyfg dwg dqg dxg dyfg dwg drg dxg dyfg dzg dog d{g d|fg dzg dpg d{g d|fg dzg dqg d{g d|fg dzg drg d{g d|fg      d}        Zjej                  j                  g d~g dwg dog dg dydfg dwg dog dg dudfg dwg dpg dg dydfg dwg dpg dg dudfg dzg dog d{g d|dfg dzg dog d{g d|dfg dzg dpg d{g d|dfg dzg dpg d{g d|dfg      d        Zkej                  j                  dmd=d gdQdQgd=d gd d=gfd d=gdQdQgd=d gd d=gfg dng ddQd gd d=gfg dsg ddQd gd d=gfg dug dd gd gfg dwg dg d{g dufg dzg dg d{g dofg d{g dg d{g dofg      d        Zld Zmej                  j                  dg d      d        Znej                  j                  dg dg dg dg dg dg      d        Zoej                  j                  dd d=gg dodfg dud dQgdfg dg dodfg dg dodfg dg ddfg      d        Zpd Zqd Zrd Zsd Ztd Zud Zvd Zw	 ddZxej                  j                  dereveuesf      ej                  j                  deewf      d               Zyd Zzej                  j                  dd      ej                  j                  dd      ej                  j                  d e{d=            d                      Z|d Z}d Z~d Zej                  j                  dg dzg dfg dzg dgfg dzgg dfg      d        Zd Zej                  j                  de5      d        Zd Zd Zd Zd Zd Zd Zd Zd Zej                  j                  dddg      d        Zd Zd Zd Zd Zej                  j                  dg dAd=dfg dAddQfg dAddfg      dƄ        Zej                  j                  d ej                  g dȢ      d=d=f ej                  g dɢ      d=dQf ej                  g dɢ      dd=f ej                  g dʢ      d=d=f ej                  g dˢ      d=dQf ej                  g dˢ      dd=fg      d̄        Zej                  j                  d ej                  g d΢      dg dAf ej                  g dy      dQg dAf ej                  g dϢ      dQg dAf ej                  g dТ      dg dѢfg      ej                  j                  dddg      dӄ               ZdԄ Zej                  j                  dg dAd=dfg dAddQfg dAdd=fg      dՄ        Zej                  j                  dg dAdfg dAdfg      d؄        Zej                  j                  dg dڢg dۢg dܢg dݢg dޢgddfg dAg dۢg dܢg dݢg dޢgddfg dg dۢg dܢg dݢg dޢgg d/dfg dg dۢg dܢg dݢg dޢgg d;dfg dg dۢg dܢg dݢg dޢgg dAdfg dg dۢg dܢg dݢg dޢgg ddfd d=gg dg dgddfg      d        Zej                  j                  de5      d        Zej                  j                  deeee g      ej                  j                  dg d      d               Zd ZddZej                  j                  deeeeg      ej                  j                  dd ej                  g d      g      d               Zd Zd Zej                  j                  dddg      d        Zej                  j                  dd d=g      d        Zd Zd Zd Zd  Zd Zy(      N)stats)datasets)make_multilabel_classification)UndefinedMetricWarning)LogisticRegression)accuracy_scoreaucaverage_precision_scoreconfusion_matrixconfusion_matrix_at_thresholdscoverage_error	dcg_score	det_curve%label_ranking_average_precision_scorelabel_ranking_lossmetric_at_thresholds
ndcg_scoreprecision_recall_curveprecision_scorerecall_scoreroc_auc_score	roc_curvetop_k_accuracy_score)_dcg_sample_scores_ndcg_sample_scores2_sort_inputs_and_compute_classification_thresholds)train_test_split)label_binarize)_sparse_random_matrix)_convert_containerassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equal)softmax)CSR_CONTAINERS)check_arraycheck_consistent_lengthcheck_random_stateFc                 ~   | t        j                         } | j                  }| j                  }|r||dk     ||dk     }}|j                  \  }}t        j                  |      }t        d      }|j                  |       ||   ||   }}t        |dz        }t
        j                  j                  d      }t
        j                  ||j                  |d|z        f   }t        d      }	|	j                  |d| |d|       j!                  ||d       }
|r	|
dddf   }
|	j#                  ||d       }||d }|||
fS )zMake some classification predictions on a toy dataset using an SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      random_state   )r   	load_irisdatatargetshapenparanger)   shuffleintrandomRandomStatec_randnr   fitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_scorey_predy_trues                O/DATA/.local/lib/python3.12/site-packages/sklearn/metrics/tests/test_ranking.pymake_predictionrN   B   sE    $$&AAQx1q51GGIz
		)A
R
 CKKNQ41qAy1}D ))


"C
a9cJ&6778A !
,Cggah%4)77$%AG !Q$-[[45"FtuXF67""    c                    t        j                  |       d   }|| |k(     }|| |k7     }|j                  dd      |j                  dd      z
  }t        j                  |dkD        }|t	        t        |      t        |      z        z  S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r0   r   )r5   uniquereshapesumfloatlen)rL   rJ   	pos_labelposnegdiff_matrix	n_corrects          rM   _aucr\   r   s     		&!!$I &I%
&C
&I%
&C++a$s{{2q'99K{Q'IuSXC0111rO   c                 `   t        j                  |       d   }t        j                  | |k(        }t        j                  |      ddd   }||   }| |   } d}t	        t        |            D ]<  }| |   |k(  sd}t	        d|dz         D ]  }| |   |k(  s|dz  } ||dz   z  }||z  }> ||z  S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r0   NrQ   r         ?)r5   rR   rT   argsortrangerV   )	rL   rJ   rW   n_posorderscoreiprecjs	            rM   _average_precisionrg      s     		&!!$IFF6Y&'EJJw"%EenGE]FE3w< !9	! D1a!e_!9	)CKD % AGODTME ! 5=rO   c                     t        | |      \  }}}t        t        |            }t        t        |            }d}t        dt	        |            D ]  }|||   ||   ||dz
     z
  z  z  } |S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r0   )r   listreversedr`   rV   )rL   rJ   	precisionrecall	thresholdaverage_precisionrd   s          rM   _average_precision_slowro      s{     $:&'#J IvyXi()I(6"#F1c)n%Yq\VAYA-FGG &rO   c                     d } || ||      \  }}t        ||      }d}|}d||z
  z  ||z   z  }	||z
  }
dd||	z
  |
|	z
  z  z   z  S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 .   t        | |      \  }}}|||k     }t        j                  ||      }|||k     }t        j                  ||kD        }|dz
  }	||	   ||   g}
||	   ||   g}t        j                  |t        j                  ||
|            }||fS )Nr0   )r   r5   appendargmaxinterp)rL   	y_predictmax_fprfprtpr_new_fprnew_tpridx_outidx_inx_interpy_interps               rM   _partial_rocz,_partial_roc_auc_score.<locals>._partial_roc   s    	2S!cWn%))GW-cWn%))C'M*1KW.KW.))GRYYw(%KL!!rO   r         ?r0   )r	   )rL   ru   rv   r   rz   r{   partial_aucfpr1fpr2min_areamax_areas              rM   _partial_roc_auc_scorer      ss    

" $FIw?GWgw'K DDdTk"dTk2Hd{H!{X-(X2EFFGGrO   dropTc                 ,   t        d      \  }}}t        ||      }t        |||       \  }}}t        ||      }t	        ||d       t        |t        ||             |j                  |j                  k(  sJ |j                  |j                  k(  sJ y )NTrA   drop_intermediater+   decimal)rN   r\   r   r	   r#   r"   r   r4   )	r   rL   ry   rJ   expected_aucrw   rx   
thresholdsroc_aucs	            rM   test_roc_curver      s     )5FAw(L$VWMCj#smGg|Q?vw!?@99		!!!99
(((((rO   c                  `   t         j                  j                  d      } t        j                  dgdz  dgdz  z         }| j	                  dd      }t        ||d      \  }}}|d   dk(  sJ |d	   dk(  sJ |j                  |j                  k(  sJ |j                  |j                  k(  sJ y )
Nr   2   r0      d   sizeTr   rQ   )r5   r9   r:   arrayrandintr   r4   )rG   rL   rK   rw   rx   thrs         rM   test_roc_curve_end_pointsr      s     ))


"CXXqcBh!r)*F[[[%FffEMCcq6Q;;r7a<<99		!!!99		!!!rO   c                  l   t        d      \  } }}t        | |      \  }}}g }|D ]I  }t        j                  ||k\  | z        }t        j                  |       }	|j	                  d|z  |	z         K t        ||d       |j                  |j                  k(  sJ |j                  |j                  k(  sJ y )NTr   r^   r+   r   )rN   r   r5   rT   rr   r#   r4   )
rL   ry   rJ   rw   rx   r   tpr_correctttprF   s
             rM   test_roc_returns_consistencyr      s     )5FAw$VW5Cj KVVW\V+,FF6N38a<(  c;:99		!!!99
(((((rO   c                      t        d      \  } }}t        j                  t              5  t	        | |       d d d        y # 1 sw Y   y xY w)NFr   )rN   pytestraises
ValueErrorr   )rL   ry   rJ   s      rM   test_roc_curve_multir      s5    (6FAw	z	"&'" 
#	"	"s   A  A	c                      t        d      \  } }}t        | |dz
        \  }}}t        ||      }t        |dd       |j                  |j                  k(  sJ |j                  |j                  k(  sJ y )NTr   r   ?r+   r   )rN   r   r	   r#   r4   )rL   ry   rJ   rw   rx   r   r   s          rM   test_roc_curve_confidencer     so    (5FAw$VWs];Cj#smGgtQ799		!!!99
(((((rO   c                     t        d      \  } }}t        j                  | j                        }t	        | |      \  }}}t        ||      }t        |dd       |j                  |j                  k(  sJ |j                  |j                  k(  sJ t        j                  | j                        }t	        | |      \  }}}t        ||      }t        |dd       |j                  |j                  k(  sJ |j                  |j                  k(  sJ t	        | |      \  }}}t        ||      }t        |dd       |j                  |j                  k(  sJ |j                  |j                  k(  sJ y )NTr   r   r+   r   g(\?)rN   r5   onesr4   r   r	   r#   zeros)rL   predrJ   trivial_predrw   rx   r   r   s           rM   test_roc_curve_hardr     sE   +48FD' 776<<(L$V\:Cj#smGgtQ799		!!!99
((((( 88FLL)L$V\:Cj#smGgtQ799		!!!99
((((( %VT2Cj#smGgtQ799		!!!99
(((((rO   c                     g d} g d}d}t        j                  t        |      5  t        | |      \  }}}d d d        t	        t        j                  t              t
        j                               |j                  j                  k(  sJ |j                  |j                  k(  sJ d}t        j                  t        |      5  t        | D cg c]  }d|z
  	 c}|      \  }}}d d d        t	        |t        j                  t        |      t
        j                               |j                  |j                  k(  sJ |j                  |j                  k(  sJ y # 1 sw Y   ,xY wc c}w # 1 sw Y   xY w)N)
r0   r0   r0   r0   r0   r0   r0   r0   r0   r0   )
r   r0   r   r0   r   r0   r   r0   r   r0   INo negative samples in y_true, false positive value should be meaninglessmatchHNo positive samples in y_true, true positive value should be meaninglessr0   )
r   warnsr   r   r$   r5   fullrV   nanr4   )rL   rK   expected_messagerw   rx   r   xs          rM   test_roc_curve_one_labelr   +  s>   +F+F 	T  
,4D	E(8S* 
F sBGGC
ORVV<=99		!!!99
((((( 	S  
,4D	E()@A!a%)@&IS* 
F sBGGC
ORVV<=99		!!!99
(((((# 
F	E *A 
F	Es)   E!	
E3E.
E3!E+.E33E<c                     ddg} ddg}t        | |      \  }}}t        | |      }t        |g d       t        |g d       t        |d       ddg} ddg}t        | |      \  }}}t        | |      }t        |g d       t        |g d       t        |d       ddg} ddg}t        | |      \  }}}t        | |      }t        |ddg       t        |ddg       t        |d       ddg} ddg}t        | |      \  }}}t        | |      }t        |g d       t        |g d       t        |d       ddg} ddg}t        | |      \  }}}t        | |      }t        |ddg       t        |ddg       t        |d       ddg} dd	g}d
}t	        j
                  t        |      5  t        | |      \  }}}d d d        t        |g d       t        |t        j                  t        j                  t        j                  g       d}t	        j
                  t        |      5  t        | |      }d d d        t        j                        sJ ddg} dd	g}d}t	        j
                  t        |      5  t        | |      \  }}}d d d        t        |t        j                  t        j                  t        j                  g       t        |g d       d}t	        j
                  t        |      5  t        | |      }d d d        t        j                  |      sJ t        j                  ddgddgg      } t        j                  ddgddgg      }t	        j
                  t        |      5  t        | |d       d d d        t	        j
                  t        |      5  t        | |d       d d d        t        t        | |d      d       t        t        | |d      d       t        j                  ddgddgg      } t        j                  ddgddgg      }t	        j
                  t        |      5  t        | |d       d d d        t	        j
                  t        |      5  t        | |d       d d d        t        t        | |d      d       t        t        | |d      d       t        j                  ddgddgg      } t        j                  ddgddgg      }t        t        | |d      d       t        t        | |d      d       t        t        | |d      d       t        t        | |d      d       t        j                  ddgddgg      } t        j                  ddgddgg      }t        t        | |d      d       t        t        | |d      d       t        t        | |d      d       t        t        | |d      d       y # 1 sw Y   )xY w# 1 sw Y   xY w# 1 sw Y   uxY w# 1 sw Y   	xY w# 1 sw Y   xY w# 1 sw Y   rxY w# 1 sw Y   xY w# 1 sw Y   xY w)Nr   r0   r   r   r0   r   r0   r0   r^           r         ?      ?r   r   r   r   r^   OOnly one class is present in y_true. ROC AUC score is not defined in that case.r   macroaverageweightedsamplesmicro)r   r   r#   r"   r   r   r   r5   r   mathisnanr   )rL   rJ   rx   rw   ry   r   r   r	   s           rM   test_roc_curve_toydatar   F  s   VF!fGFG,KCaFG,Gc9-c9-%VF!fGFG,KCaFG,Gc9-c9-%VF!fGFG,KCaFG,GcAq6*cAq6*%VF!fGFG,KCaFG,Gc9-c9-%VFCjGFG,KCaFG,GcAq6*cAq6*% VFTlG 	S  
,4D	E0S! 
Fc?3cBFFBFFBFF#;<	5  
,4D	EFG, 
F::c?? VFTlG 	T  
,4D	E0S! 
FcBFFBFFBFF#;<c?3	5  
,4D	EFG, 
F::c?? XX1v1v&'FhhAA'(G	,4D	Efgw7 
F	,4D	Efgz: 
FfgyI3OfgwGMXX1v1v&'FhhAA'(G	,4D	Efgw7 
F	,4D	Efgz: 
FfgyI3OfgwGMXX1v1v&'FhhAA'(GfgwGKfgzJANfgyI1MfgwGKXX1v1v&'Fhhc
S#J/0GfgwGMfgzJCPfgyI3OfgwGM} 
F	E 
F	E 
F	E 
F	E 
F	E	E	E 
F	E	E	Es`   "WW'0W4-X*XX4X(&X5W$'W14W>XXX%(X25X?c            	          g d} g d}t        | |d      \  }}}t        |t        j                  dddg       g d} g d	}t        | |d      \  }}}t        |t        j                  dd
dddg       y )Nr   r   r   r   r0   r0   r   皙?r   333333?ffffff?r^   Tr   r^   r   r   r   r   r   r   r   r   r   r0   r0   r0   r0   r0   r0   r   皙?r   r   r   皙?r   r   r   r   r   r   r^   r   r   )r   r#   r5   inf)rL   rJ   rx   rw   r   s        rM    test_roc_curve_drop_intermediater     st    F,G$VWMCjj2663S*AB 5FOG$VWMCjj2663S#s*KLrO   c                     g d} g d}t        j                  dd      }t        | ||      \  }}}t        j                  |      dk  j	                         dk(  sJ t        j                  |      dk  j	                         dk(  sJ y )Nr   r   r0   r0   r0   )r   r   333333?皙?r   r      sample_weightr   )r5   repeatr   diffrT   )rL   rJ   r   rw   rx   ry   s         rM   !test_roc_curve_fpr_tpr_increasingr     sz     F'GIIc1%MFG=IKCaGGCL1!!#q(((GGCL1!!#q(((rO   c                  0   ddg} ddg}t        t        | |      d       ddg} ddg}t        t        | |      d       g d} g d}t        t        | |      d       ddg} ddg}t        t        | |      d       g d} g d}t        t        | |      d       y )Nr   r0   r   r0   r   r   r   r   r   r0   )r#   r	   )r   rC   s     rM   test_aucr     s    	
AA	
AAc!Qi-	
AA	
AAc!Qi-AAc!Qi-	
AA	
AAc!Qi+AAc!Qi-rO   c                     t        j                  t              5  t        g dddg       d d d        t        j                  t              5  t        dgdg       d d d        g d} g d}dj	                  t        j                  |             }t        j                  t        t        j                  |            5  t        | |       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)	Nr   r   r   r   )r+   r0   r      )r            z+x is neither increasing nor decreasing : {}r   )	r   r   r   r	   formatr5   r   reescape)r   rC   error_messages      rM   test_auc_errorsr     s    	z	"Oc3Z( 
# 
z	"SEC5 
# 	AAAHHRSUM	z=)A	BAq	 
C	B 
#	" 
#	" 
C	Bs#   CC=C+CC(+C4zy_true, labels)r   r0   r   r+   r   r0   r+   )abr   c)r   r   r   c           	      v   t        j                  g dg dg dg dg      }t        g dg d      }t        g dg d      }||z   d	z  }t        g d
g d      }t        g dg d      }||z   d	z  }t        ddgddg      }	t        ddgddg      }
|	|
z   d	z  }||z   |z   dz  }t        t        | ||d      |       |||g}g d}t        j                  ||      }t        t        | ||dd      |       d}t        j                  t        |      5  t        | ||dd        d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   ffffff?r   333333?r   r   r   r0   r   r0   )r   r   r   r   r0   r   )r   r   r   r+   r0   r0   r   )r   r   r   r   )r   r   r   r0   r   r   r   r   r   r   ovolabelsmulti_class)r   r   r   )weightsr   r   r  r   z6average=None is not implemented for multi_class='ovo'.r   )r5   r   r   r"   r   r   r   NotImplementedError)rL   r   y_scoresscore_01score_10average_score_01score_02score_20average_score_02score_12score_21average_score_12ovo_unweighted_scorepair_scores
prevalenceovo_weighted_scorer   s                    rM   #test_multiclass_ovo_roc_auc_toydatar    s]    xx	/+<mLH Y(89HY8H 8+q0 Y7HY(89H 8+q0 aVc3Z0HaVc3Z0H 8+q0 -/??BRRVWWfhv5I $%57GHK#JKDHV
	
 		 MM	*-	@fhv5RVW 
A	@	@s   D//D8)r   r+   r   r+   )r   dr   r  )r   r   r  c           	          t        j                  g dg dg dg dg      }t        g dg d      }t        g dg d      }||z   d	z  }t        t        | ||d
      |       t        t        | ||d
d      |       y )N)r   r   r   )r   r   r   )皙?r   ?)r   r   r   )r0   r   r0   r   )r   r   r  r   r   r0   r   r0   )r   r   r  r   r+   r   r   r   r  r5   r   r   r"   )rL   r   r  r  r  	ovo_scores         rM   *test_multiclass_ovo_roc_auc_toydata_binaryr  1  s     xx	/+<oNH \+@AH\+@AHH$)Ifhv5I9
 HV
	
 		rO   )r   r0   r+   r+   )r   r   r   r   c           	         t        j                  g dg dg dg dg      }t        g d|d d df         }t        g d|d d df         }t        g d	|d d d
f         }t        t        | |d|d       |||g       ||z   |z   dz  }t        t        | |d|      |       |dz  |dz  z   |dz  z   }t        t        | |d|d      |       y )Nr^   r   r   )r   r   r   )r   r   r   )r   r   r   r0   r   r   r   r   )r   r0   r   r   r0   r   r   r0   r0   r+   ovr)r  r   r   g      @)r  r   r   r   r   r  )rL   r   r  out_0out_1out_2result_unweightedresult_weighteds           rM   #test_multiclass_ovr_roc_auc_toydatar&  U  s     xx	/?OLH
 ,A7E,A7E,A7EfhE&RVW	u .#5fhE&I dlUT\1ECK?OH%
	
 		rO   zmulti_class, average))r   r   )r   r   )r   r   c                 B   t        j                  g d      }g dg dg dg dg}t        t        ||| |      d       g dg dg dg dg}t        ||| |      dk  sJ dt        j                  d	      z  }t        ||| |      t        j                  d
      k(  sJ y )N)r   r0   r+   r   )r   r   r   r^   )r   r^   r   r   )r   r   r^   r   )r   皙?r(  r   r  r   r^   r   )r   r   r   )r5   r   r"   r   r   r   approx)r  r   rL   	y_perfecty_imperfecty_chances         rM   0test_perfect_imperfect_chance_multiclass_roc_aucr.    s     XXl#F 	 	I fi['R 		K 	fk{GT
		
 bggfo%Hk7	s	  rO   c           
         | }t         j                  j                  g dd|      }t        j                  |D cg c]2  }t         j
                  j                  d||      j                         4 c}      }t        |g d      }t        |j                         |j                               \  }}}t        ||      }	t        ||dd	
      }
|	t        j                  |
      k(  sJ y c c}w )N)       @r^   r     )r   r/   r0   )nrF   r/   r   )classesr   r   r)  )r   	dirichletrvsr5   asarraymultinomialrs   r   r   ravelr	   r   r   r*  )global_random_seedseedrK   y_pred_irL   y_onehotrw   rx   ry   roc_auc_by_handroc_auc_autos              rM   test_micro_averaged_ovr_roc_aucr?    s    D
 __  t$ OFZZ #	
" !!A!ELLN"	
F fi8HHNN,flln=KCa#smO UGTLfmmL9999	
s   7C)zmsg, y_true, labelsz!Parameter 'labels' must be uniquer   r+   r   )r   r   r   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)r   r   r   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r0   r   r   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r0   r+   r   )r   r   r   r  z2'y_true' contains labels not in parameter 'labels')r   r   r   er  r   r   c                     t        j                  g dg dg dg dg      }t        j                  t        |       5  t        ||||       d d d        y # 1 sw Y   y xY w)Nr   r   r   r   r   r   )r5   r   r   r   r   r   )msgrL   r   r  r  s        rM   *test_roc_auc_score_multiclass_labels_errorrD    sJ    R xx	/+<mLH 
z	-fhv;O 
.	-	-s   AAzmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsr   )r   r  zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)r  r   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadr   )r  rv   z'multi_class must be in \('ovo', 'ovr'\)c                     t        d      }|j                  dd      }t        |      }|j                  ddd      }t	        j
                  t        |       5  t        ||fi | d d d        y # 1 sw Y   y xY w)N     r   r   r   r   )r)   randr%   r   r   r   r   r   )rC  kwargsrG   rJ   y_probrL   s         rM   #test_roc_auc_score_multiclass_errorrK    se    N S
!Chhr1oGWF[[AB['F	z	-ff// 
.	-	-s   A//A8c                  4   t        d      } | j                  d      }t        j                  dd      }d}t	        j
                  t        |      5  t        ||       d d d        t        j                  dd      }t	        j
                  t        |      5  t        ||       d d d        t        j                  ddd      }t	        j
                  t        |      5  t        ||       d d d        y # 1 sw Y   xY w# 1 sw Y   ^xY w# 1 sw Y   y xY w)NrF  
   r8   dtyper   r   rQ   )
r)   rH  r5   r   r   r   r   r   r   r   )rG   rK   rL   warn_messages       rM   test_auc_score_non_binary_classrQ  >  s     S
!CXXb\FXXb&F	5  
,L	Aff% 
BWWRu%F	,L	Aff% 
BWWR5)F	,L	Aff% 
B	A 
B	A 
B	A 
B	As$   C6D D6C?DDc                     t        j                  g d      } t        j                  ddg      }t        j                  t        d      5  t        | |       ddd       t        j                  g d      } t        j                  dt         j                  dd	g      }t        j                  t        d
      5  t        | |       ddd       y# 1 sw Y   xxY w# 1 sw Y   yxY w)zKTest `_sort_inputs_and_compute_classification_thresholds` input validation.r   r   r   inconsistent numbers of samplesr   Nr  r   r   zInput.*contains NaN)r5   r   r   r   r   r   r   rL   rJ   s     rM   Gtest_sort_inputs_and_compute_classification_thresholds_input_validationrU  S  s     XXi FhhSz"G	z)J	K:67K 
L XXl#FhhRVVS#./G	z)>	?:67K 
@	? 
L	K 
@	?s   
C8CCC#c                  Z   t        j                  g d      } t        j                  g d      }t        j                  g d      }t        | ||      \  }}}}t        |      t        |      cxk(  rt        |      cxk(  rdk(  sJ  J d|vsJ d|vsJ t        | |      \  }}}}|J t        j                  g d      } t        j                  g d	      }t        j                  g d
      }t	        j
                  t        d      5  t        | ||       ddd       y# 1 sw Y   yxY w)zJTest zero weights in `_sort_inputs_and_compute_classification_thresholds`.r   r0   r   r0   r   r0   )r   r   r   r   r   r   )r   r0  r^         ?r   r   r   r   r   Nr   )r   r   r   )r   r   r   z$Sample weights must contain at leastr   )r5   r   r   rV   r   r   r   )rL   rJ   r   y_true_sortedy_score_sortedweight_sortedry   weights           rM   Ctest_sort_inputs_and_compute_classification_thresholds_zero_weightsr]  d  s   XX()Fhh56GHH;<M 	;G]	
 4M>=! }^!4OM8JOaOOOOOn$$$n$$$ IOAq&! >> XXi Fhh'GHH_-M	z)O	P:G]	
 
Q	P	Ps   
D!!D*c                  J   t        j                  g d      } t        j                  g d      }t        j                  g d      }t        | ||      \  }}}}t        j                  |dd |dd k\        sJ t	        |t        j                  g d             t	        |t        j                  g d             t	        |d	d
 ddg       t	        |d	d
 ddg       t        j                  g d      }t        | |      \  }}}}t	        |t        j                  d
g             y)zETest sorting in `_sort_inputs_and_compute_classification_thresholds`.)r   r0   r   r0   r0   )r   r   r   r   r   )r^   r0  rX  r   r   NrQ   r0   )r0  r   rX  r   r^   )r   r0   r   r   r+   r   r   r   )r   r   r   r   r   )r5   r   r   allr$   )	rL   rJ   r   rY  rZ  r[  threshold_idxsy_score_samery   s	            rM   >test_sort_inputs_and_compute_classification_thresholds_sortingrb    s   XXo&Fhh01GHH67M 	;G]	
 AM>=. 66."%);;<<<}bhh/H&IJ~rxx'=>~a*S#J7}Qq)Aq62 8856LPAq!^ ~rxx}5rO   c                    t         j                  j                  |       }d}|j                  ddd      }|j	                  d      }t        j
                  |      }||z
  }t        ||      \  }}}	}
}t        |      t        |      cxk(  r-t        |	      cxk(  rt        |
      cxk(  rt        |      k(  sJ  J t        |
|	z   |       t        ||z   |       t        ||z   |	z   |
z   |       y)z.Smoke test for confusion_matrix_at_thresholds.r   r   r+   r   N)	r5   r9   r:   r   uniformrT   r   rV   r!   )r9  rG   rD   rL   rJ   ra   n_negtnsfpsfnstpsr   s               rM   #test_confusion_matrix_at_thresholdsrj    s    
))

 2
3CI[[AC[(Fkksk#GFF6NEE%CFG%T"Cc3
s8s3xJ3s8Js3xJ3z?JJJJJC#Iu%C#Iu%C#IOc)95rO   
curve_funcc                     t        d      }|j                  ddd      }|j                  d      }d}t        j                  t
        |      5   | ||       d d d        y # 1 sw Y   y xY w)NrF  r   r   rM  r   z"multiclass format is not supportedr   )r)   r   rH  r   r   r   )rk  rG   rL   rK   rC  s        rM   4test_confusion_matrix_at_thresholds_multiclass_errorrm    sV    
S
!C[[AB['FXXb\F
.C	z	-66" 
.	-	-s   
A!!A*c                    d}t        j                  t        |      5   | t        j                  ddgd      ddg       d d d        t        j                  t        |      5   | t        j                  ddgt
              ddg       d d d        g d	} | g d
|      } | g d|      }t        ||      D ]%  \  }}t        j                  j                  ||       ' y # 1 sw Y   xY w# 1 sw Y   dxY w)Nzy_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r   r   r   z<U1rN  r   r^   )r   r^   r   gzG?r   r0   r0   r   )r   r^   r^   r   )	r   r   r   r5   r   objectziptestingr!   )rk  rC  rK   	int_curvefloat_curveint_curve_partfloat_curve_parts          rM   6test_confusion_matrix_at_thresholds_implicit_pos_labelrw    s    
	  
z	-288S#Je4sCjA 
. 
z	-288S#Jf5SzB 
.
 #F<0I16:K,/	;,G((


"">3CD -H 
.	- 
.	-s   #C"$'C."C+.C7z.ignore:Support for labels represented as byteslabels_typeri   r   c                     t        ddg|      }d}t        j                  t        |      5   | |ddg       d d d        y # 1 sw Y   y xY w)N   a   bz8Support for labels represented as bytes is not supportedr   r   r^   )r    r   r   	TypeError)rk  rx  r   rC  s       rM   <test_confusion_matrix_at_thresholds_implicit_bytes_pos_labelr}    sA      tk:F
DC	y	,6C:& 
-	,	,s   AA
c                     g d}g d}g d} | |||      } | |d d |d d |d d       }t        ||      D ]  \  }}t        ||        y )Nr   r   r   r   r   r   )r0   r0   r0   r   r   r   rQ   )rq  r!   )rk  rL   rJ   r   result_1result_2arr_1arr_2s           rM   6test_confusion_matrix_at_thresholds_zero_sample_weightr    sa    F'G%M&'GH&"+ws|=QTRTCUVHHh/uu% 0rO   c                    t        d      \  }}}t        |||        t        |dd  |dd  |       \  }}}|d   dk(  sJ |d   |dd  j                         k(  sJ d|t	        j
                  |dk(        <   |j                         }t        |||        t        ||       g d}g d	}	t        ||	|       \  }}}| r+t        |g d
       t        |g d       t        |g d       n*t        |g d       t        |g d       t        |g d	       |j                  |j                  k(  sJ |j                  |j                  dz   k(  sJ y )NTr   r0   r   r   r^   rQ   r0   r   r   r0   )r0   r+   r   r   )r   QUU?r^   r^   )r^   r   r   r   )r0   r+   r   )r   r  r   r^   r^   )r^   r   r   r   r   )
rN   _test_precision_recall_curver   meanr5   wherecopyr$   r!   r   )
r   rL   ry   rJ   rF   rr   y_true_copyr   predict_probass
             rM   test_precision_recall_curver    s<   (5FAw $7 %VABZPTUGAq!Q43;;Q46!":??$$$$ %'F288FaK !++-K $7{F+F!N$V^tTGAq!67/09%;<45<(66QVV66QVVaZrO   c                    t        | ||      \  }}}t        | |      }t        |dd       t        |t        | |             t	        t        | |      |d       |j                  |j                  k(  sJ |j                  |j                  dz   k(  sJ t        | t        j                  |      |      \  }}}|j                  |j                  k(  sJ |j                  |j                  dz   k(  sJ y )Nr   g+?r   r+   r   r0   )	r   ro   r#   r
   r"   rg   r   r5   
zeros_like)rL   rJ   r   rF   r  r   precision_recall_aucs          rM   r  r    s    -fgQUVAq*267C2E1=5fgF 67+-A1 66QVV66Z__q((((-g&$Aq* 66QVV66Z__q((((rO   c           	      h   t        j                  d      5  ddg}ddg}t        |||       \  }}}t        ||      }t	        |g d       t	        |g d       t        |d       ddg}ddg}t        |||       \  }}}t        ||      }t	        |g d	       t	        |g d
       t        |d       ddg}ddg}t        |||       \  }}}t        ||      }t	        |ddg       t	        |ddg       t        |d       ddg}ddg}t        |||       \  }}}t        ||      }t	        |g d       t	        |g d       t        |d       ddg}ddg}t        |||       \  }}}t        ||      }t	        |ddg       t	        |ddg       t        |d       ddg}ddg}t        j                  t        d      5  t        |||       \  }}}d d d        t        j                  t        d      5  t        ||      }d d d        t        |g d       t        |g d       t        |d       ddg}ddg}t        |||       \  }}}t        t        ||      d       t	        |g d       t	        |g d       t        j                  ddgddgg      }t        j                  ddgddgg      }t        j                  t        d      5  t        t        ||d      d       d d d        t        j                  t        d      5  t        t        ||d      d       d d d        t        t        ||d      d       t        t        ||d      d       t        j                  ddgddgg      }t        j                  ddgddgg      }t        j                  t        d      5  t        t        ||d      d       d d d        t        j                  t        d      5  t        t        ||d      d       d d d        t        t        ||d      d       t        t        ||d      d       t        j                  ddgddgg      }t        j                  ddgddgg      }t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        j                  ddgddgg      }t        j                  ddgddgg      }t        j                  t        d      5  t        t        ||d      d       d d d        t        t        ||d      d       t        j                  t        d      5  t        t        ||d      d       d d d        t        j                  t        d      5  t        t        ||d      d       d d d        t        j                  ddgddgg      }t        j                  ddgddgg      }t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        j                  ddgddgg      }t        j                  ddgddgg      }t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       d d d        t        j                  d      5  t        j                  ddgddgg      }t        j                  ddgddgg      }t        j                  t        d      5  t        t        ||d      d       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   gxY w# 1 sw Y   8xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)Nraise)r_  r   r0   r   )r   r0   r0   r   r^   )r   r   r^   r  r   r   r   r   z!No positive class found in y_truer   r   )r^   r^   r^   )r0   r   r   r   r   r   r   r   ignore)r5   errstater   r
   r#   r"   r   r   UserWarningr!   r   )r   rL   rJ   rF   r  ry   auc_prcs          rM   #test_precision_recall_curve_toydatar  )  sv   		!Qa&(DQ1a)&':!![1!!Y/GS)Qa&(DQ1a)&':!!_5!!_5 	GS)Qa&(DQ1a)&':!!c1X.!!c1X.GS)Qa&(DQ1a)&':!![1!!Y/GS)Q*(DQ1a)&':!!c1X.!!aX.GS)Q,\\+-PQ,VWPTUGAq! R\\+-PQ-fg>G R9%9%#Q,(DQ1a3FGDcJ!!_5!!]3 Aq6Aq6*+((QFQF+,\\+-PQ'I3 R \\+-PQ'Lc R 	#FGYG	
 	/QSVWAq6Aq6*+((QFQF+,\\+-PQ'I3 R \\+-PQ'Lc R 	#FGYG	
 	/QSVWAq6Aq6*+((QFQF+,#FGWEs	
 	#FGZH#	
 	#FGYG	
 	#FGWEs	
 Aq6Aq6*+((QFQF+,\\+-PQ'I3 R 	#FGZH#	
 \\+-PQ'KS R \\+-PQ'I3 R
 Aq6Aq6*+((QFQF+,/QSVW#FGZH#	
 	#FGYG	
 	/QSVWAq6Aq6*+((S#Jc
34#FGWEs	
 	#FGZH#	
 	#FGYG	
 	#FGWEs	
i 
"p 
	"Aq6Aq6*+((QFQF+,\\+-PQ'La R	 
#	"S RQQQ  RQ RQ RQ RQ4 RQ RQ RQs 
"	!x RQ	 
#	"s  F*`^#`7^'C`^4*#`_&B	`/_#`+_D`#_(<;`7_5#`3`D4`A`(0`	`(^$	`'^1	,`4^>	9`_	`_	`_%	 `(_2	-`5_?	:``	```%	!`((`1c                  D   g d} g d}t        | |d      \  }}}t        |g d       g d} g d}t        | |d      \  }}}t        |g d       g d	} g d
}t        | |d      \  }}}t        |ddg       g d} g d
}t        | |d      \  }}}t        |g d
       y)z9Check the behaviour of the `drop_intermediate` parameter.r   r   Tr   )r   r   r^   r   r   )r   r   r   r   r   r^   r   r   r   r   )r   r   r   r   r   r   r0   r0   r0   r0   N)r   r!   )rL   rJ   rk   rl   r   s        rM   -test_precision_recall_curve_drop_intermediater    s    F,G$:4%!Ivz J0 5FOG$:4%!Ivz J >? F"G$:4%!Ivz Jc
+ F"G$:4%!Ivz J 45rO   c                      t        j                  dt              } d| d d d<   t        j                  d      }t	        | |      dk(  sJ y )Nr   rN  r0   r   r   )r5   r   r8   r   r
   rT  s     rM   &test_average_precision_constant_valuesr    sE    
 XXc%FF3Q3KggclG #673t;;;rO   c                     t        j                  ddg      } t        j                  ddg      }t        j                  d      }t	        j
                  t        |      5  t        | |d       d d d        y # 1 sw Y   y xY w)Nr   r0   z;pos_label=2 is not a valid label. It should be one of [0 1]r   r+   rW   )r5   r   r   r   r   r   r   r
   rL   rK   err_msgs      rM   4test_average_precision_score_binary_pos_label_errorsr    s[    XXq!fFXXq!fFiiUVG	z	1!< 
2	1	1s   A77B c                     t        j                  ddgddgddgddgg      } t        j                  ddgddgddgddgg      }d}t        j                  t        |      5  t        | |d	       d d d        y # 1 sw Y   y xY w)
Nr0   r   r   r   r   r   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r   r  r5   r   r   r   r   r
   r  s      rM   8test_average_precision_score_multilabel_pos_label_errorsr    s     XX1v1v1v1v67FXXSzC:SzC:FGF	6  
z	1!< 
2	1	1s    A88Bc            	          t        j                  g d      } t        j                  g dg dg dg dg dg dg      }d}t        j                  t        |      5  t        | |d	       d d d        y # 1 sw Y   y xY w)
Nr   r0   r+   r   r0   r+   )r   r   r   )r   r   r   )r   r   r   )r   r   r   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r   r   r  r  )rL   rJ   r  s      rM   8test_average_precision_score_multiclass_pos_label_errorsr    sh    XX()Fhh	
	G	6  
z	11= 
2	1	1s   A44A=c                     t        j                  g d      } t        j                  g d      }t        j                  d      }t	        j
                  t        |      5  t        | |       ddd       t	        j
                  t        |      5  t        | |d       ddd       t	        j
                  t        |      5  t        | |       ddd       y# 1 sw Y   lxY w# 1 sw Y   FxY w# 1 sw Y   yxY w)zNTest ranking metrics, with multiclass support, raise if shape `y_score` is 1D.r  )r   r   r   r   r   r   z7`y_score` needs to be of shape `(n_samples, n_classes)`r   Nr   )r  )
r5   r   r   r   r   r   r   r
   r   r   )rL   rJ   rC  s      rM   Dtest_multiclass_ranking_metrics_raise_for_incorrect_shape_of_y_scorer  (  s    XX()Fhh56G
))M
NC	z	-0 
.	z	-fg59 
.	z	-VW- 
.	-	 
.	-	-	-	-	-s$   CC#C/C #C,/C8c                  \   t        d      \  } }}t        | |      }t        | d|z        }t        | d|z        }t        | |dz
        }||k(  sJ ||k(  sJ ||k(  sJ t        | |      }t        | d|z        }t        | d|z        }	t        | |dz
        }
||k(  sJ ||	k(  sJ ||
k(  sJ y )NTr   r   gư>rM  )rN   r   r
   )rL   ry   rJ   r   roc_auc_scaled_uproc_auc_scaled_downroc_auc_shiftedpr_aucpr_auc_scaled_uppr_auc_scaled_downpr_auc_shifteds              rM   test_score_scale_invariancer  6  s     )5FAwFG,G%fcGm<'w?#FGbL9O''''))))o%%%$VW5F.vsW}E0H,VWr\BN%%%%''''^###rO   z(y_true,y_score,expected_fpr,expected_fnrr   r   )r   r   r   )r   r   r0   r   r   r   r   )r   r   r   r   r   r   )r0   r0   r   r   )r   r0   r0   r0   r   r   )r   r   r   c                 T    t        | |      \  }}}t        ||       t        ||       y Nr   r!   rL   rJ   expected_fprexpected_fnrrw   fnrry   s          rM   test_det_curve_toydatar  O  s*    6 FG,KCaC&C&rO   )rL   rJ   r  r  r   )r0   r0   r   r   )r0   r0   r   c                 X    t        | ||      \  }}}t        ||       t        ||       y )Nr   r  )rL   rJ   r  r  r   rw   r  ry   s           rM    test_det_curve_drop_intermediater  p  s-    & FG?PQKCaC&C&rO   r   r   r   c                 T    t        | |      \  }}}t        ||       t        ||       y r  r  r  s          rM   test_det_curve_tie_handlingr    s*     FG,KCaC&C&rO   c                  T    t        t        g dg d      t        g dg d             y )Nr   r   r   )r   r   r   r   r0   r0   )r!   r    rO   rM   test_det_curve_sanity_checkr    s     )[)$&<=rO   rJ   )r   r   r   r   r0   c                     t        g dt        j                  d|             \  }}}t        |ddg       t        |ddg       t        || t        j                  g       y )NrW  r   rT  r0   r   )r   r5   r   r!   r   )rJ   rw   r  rm   s       rM   test_det_curve_constant_scoresr    sT    #!2771g+>Ci C!Q C!Q I01rO   rL   )r   r   r   r   r   r0   r   )r   r   r   r0   r0   r0   )r   r   r0   r0   r0   r0   )r   r0   r0   r0   r0   r0   c                 Z    t        | |       \  }}}t        |dg       t        |dg       y )NrT  r   r  )rL   rw   r  ry   s       rM   test_det_curve_perfect_scoresr    s.     66:KCaC!C!rO   zy_true, y_pred, err_msgrS  r   r   r   z#Only one class is present in y_truer0   r0   r0   )cancerr  
not cancer)r   r   r   zpos_label is not specifiedc                 |    t        j                  t        |      5  t        | |       d d d        y # 1 sw Y   y xY w)Nr   )r   r   r   r   r  s      rM   test_det_curve_bad_inputr    s(      
z	1&&! 
2	1	1   2;c                  Z   dgdz  dgdz  z   } t        j                  g d      }d|z
  }t        | |d      \  }}}t        | |d      \  }}}|d   t        j                  d	      k(  sJ |d   t        j                  d
      k(  sJ t        ||d d d          t        ||d d d          y )Nr  r   r  r   )
r   r   r   r   r   r   r   r   r   r   r0   r  r   r   r   rQ   )r5   r   r   r   r*  r!   )	rL   y_pred_pos_not_cancery_pred_pos_cancerfpr_pos_cancerfnr_pos_cancerth_pos_cancerfpr_pos_not_cancerfnr_pos_not_cancerth_pos_not_cancers	            rM   test_det_curve_pos_labelr    s    Z!^|nq00FHH%WX114=51NNM
 AJA=*,= v}}S1111Q6==#5555 N$6tt$<=N$6tt$<=rO   c                    t         | ddggddgg      d       t         | ddggddgg      d       t         | ddggddgg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g d	gg dg      d       t         | g d
gg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g d	gg dg      d       t         | g d
gg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g d	gg dg      d       t         | g d
gg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | ddggddgg      d       t         | ddggddgg      d       t         | ddggddgg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g d	gg dg      d       t         | g d
gg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       t         | g dgg dg      d       y )Nr   r0   r   r   r   r   r  r   r   r   gUUUUUU?r   g?r   g?r   r   r   r  r   r   r   r  )r   r   r   UUUUUU?)r0   r0   r0   r   )r   r   r   r   )r"   
lrap_scores    rM   check_lrap_toyr    sp   
QF8tTl^<a@
QF8tTl^<eD
QF8tTl^<a@
I;1B0CDaH
I;1B0CDeL
I;1B0CDaH
I;1B0CDeLI;!2 346I I;!2 346I 
I;1B0CDeL
I;1B0CDeLI;!2 346I 
I;1B0CDaH
I;1B0CDoV
I;1B0CDaH
I;1B0CDaH
I;1B0CDeL
I;1B0CDaH
I;1B0CDoV
I;1B0CDeLI;!2 346I 
I;1B0CDaH
I;1B0CDaH 
QF8sCj\:C@
QF8sCj\:C@
QF8sCj\:A>
I;1A0BCSI
I;1A0BCSI
I;1A0BCQG
I;1A0BCUKI;!1 235H I;!1 235H 
I;1A0BCQG
I;0ABEJ
L>4H3IJERrO   c           	         t        d      }t        dd      D ]  }|j                  d|f      }t        j                  |      }t        j
                  d|f      } | ||      dk(  sJ  | ||      dk(  sJ t        j                  d|f      } | ||      dk(  sJ  | ||      dk(  rJ  t         | dgdgdgdggdgdgdgdgg      d       y )Nr   r+   r   r0   r   r^   r   )r)   r`   rd  r5   r  r   r   r"   )r  r/   n_labelsrJ   y_score_tiesrL   s         rM   !check_zero_or_all_relevant_labelsr  *  s    %a(L!QK&&QM&:}}W- 1h-(&'*c111&,/3666 !X'&'*c111&,/3666   QC!qcA3'3%#u)EFrO   c                    t        j                  t              5   | g dg d       d d d        t        j                  t              5   | g dg dg dg dg       d d d        t        j                  t              5   | g dg dg dg dg       d d d        t        j                  t              5   | ddgddggddg       d d d        t        j                  t              5   | ddgddggddgg       d d d        t        j                  t              5   | ddgddggdgdgg       d d d        t        j                  t              5   | ddggddgddgg       d d d        t        j                  t              5   | dgdggddgddgg       d d d        t        j                  t              5   | ddgddggdgdgg       d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   _xY w# 1 sw Y   9xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)	Nr   )r   r   r   r   )r   r   r   )r   r   r   )r   r   r   r   r0   )r   r   r   r  s    rM   check_lrap_error_raisedr  A  s   	z	"9./ 
#	z	"90/?ST 
#	z	"//R	
 
# 
z	"QFQF#aV, 
#	z	"QFQF#q!fX. 
#	z	"QFQF#qcA3Z0 
#	z	"QF8q!fq!f-. 
#	z	"QC!:AA/0 
#	z	"QFQF#qcA3Z0 
#	") 
#	"	"	"	"	" 
#	"	"	"	"	"	"	"	"	"	"	"sk   G	G$?G15G>(HHH$H0:H<G!$G.1G;>HHH!$H-0H9<Ic           	         t        dd      D ]v  }t        j                  d|f      }t        d|      D ]N  }t        ||z
        D ];  }t        j                  d|f      }d|d|||z   f<   t	         | ||      ||z         = P x y )Nr+   rM  r0   r   )r`   r5   r   r   r"   )r  r  rJ   
n_relevantrX   rL   s         rM   check_lrap_only_tiesr  [  s     !RL''1h-(  8,JX
231h-045q#j 0001#Jvw$?hAVW 4 -	 !rO   c                    t        dd      D ]  }|t        j                  |      j                  d|f      dz   z
  }t        j                  d|f      }d|d<   d|d<   t         | ||      d|z  dz   dz         t        d|      D ]g  t        |z
        D ]T  t        j                  d|f      }d|dz   f<   t         | ||      t        fdt              D                     V i  y )Nr+   rM  r0   )r   r   )r   rQ   r   c              3   @   K   | ]  }|d z   |z   d z   z  z    yw)r0   Nr  ).0r  r  rX   s     rM   	<genexpr>z>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>~  s/      !2A QC!GaK:#=>!2s   )r`   r5   r6   rS   r   r"   rT   )r  r  rJ   rL   r  rX   s       @@rM   +check_lrap_without_tie_and_increasing_scorer  j  s     !RLbii1991h-H1LM 1h-(tuJvw7!h,:Ja9OP  8,JX
231h-045q#j 0001#vw/ !&z!2  4 - !rO   c                    t        | |       t        |       } t        |      }| j                  \  }}t        j                  |f      }t        |      D ]  }t        j                  ||   d      \  }}|j                  }||z
  t        j                  |dz         j                         }	|	   | |   j                         d   }
|
j                  dk(  s|
j                  |k(  rd||<   d||<   |
D ]*  t        fd|
D              }||xx   |   z  z  cc<   , ||xx   |
j                  z  cc<    |j                         S )z8Simple implementation of label ranking average precisionT)return_inverser0   )	minlengthr   r   c              3   4   K   | ]  }|      k    y wr  r  )r  r  labelranks     rM   r  z_my_lrap.<locals>.<genexpr>  s      JAaDK!7s   )r(   r'   r4   r5   emptyr`   rR   r   bincountcumsumnonzerorT   r  )rL   rJ   rD   r  rc   rd   unique_rankinv_rankn_ranks	corr_rankrelevantn_ranked_abover  r  s               @@rM   _my_lrapr    s@   FG, F'"G ,,IxHHi\"E9 !#		'!*T JX""! KK!<CCE	!9$$&q)==A(!:E!HaE ! J JJN !He44H  	aHMM!5 8 ::<rO   r   c                    t        dd|||      \  }}t        |j                  d   |j                  d   |      }t        |d      r|j	                         }t        ||      }t        ||      }t        ||       t        |      }|j                  ||f      }t        ||      }t        ||      }t        ||       y )Nr0   F)rE   allow_unlabeledr/   	n_classesrD   r   )n_componentsrE   r/   toarrayr   )
r   r   r4   hasattrr  r   r  r"   r)   rd  )	r  r  rD   r/   ry   rL   rJ   
score_lrapscore_my_lraps	            rM   %check_alternative_lrap_implementationr
    s     /!IAv $\\!_<<?!G w	"//#6vwGJVW-M
M2 &l3L""I(>"?G6vwGJVW-M
M2rO   checkfuncc                      | |       y r  r  )r  r  s     rM   test_label_ranking_avpr    s     
$KrO   c                  "    t        t               y r  )r  r   r  rO   rM   test_lrap_error_raisedr    s    ABrO   rD   )r0   r+   r   rG  r  )r+   r   rM  r/   c                 (    t        t        || |       y r  )r
  r   )rD   r  r/   s      rM   $test_alternative_lrap_implementationr    s     *-y)\rO   c                  l   t        j                  g dg dg dgt              } t        j                  g dg dg dg      }t        j                  g d      }t        j                  g d	      }t        t	        | ||
      t        j
                  ||z        t        j
                  |      z         y )Nr  r  r  rN  )r   r   r   r   )r   r   r   r   r   r   r   r   )r   r   r^   )r^   r^   r   r   )r5   r   boolr"   r   rT   )rL   rJ   samplewise_lrapsr   s       rM   &test_lrap_sample_weighting_zero_labelsr    s     XX|\<@MFhh	35IJG xx 01HH_-M-G=	
 	}//0266-3HH	rO   c            	         t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	gg dg      d       t        t        g d
gg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	gg dg      d       t        t        g d
gg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	gg dg      d       t        t        g d
gg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	g dgg dg dg      d       t        t        g d	g dg d
gg dg dg dg      d       t        t        g d	g dg d
gg dg dg dg      d       y )Nr   r0   r   r   r+   r  r  r   r   r   r   r   r   r   r  r  r  r   g      $@r   r0   r   r0  r   rM  r  r@  g@r   r0   r   r"   r   r  rO   rM   test_coverage_errorr    s   Ax4,@!DAx4,@!DAx4,@!DAx4,@!D	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L	{5F4GH!L 	9-/KL
 	9-y)/T	
 		 	9-y)/T	
 		rO   c                     t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	gg dg      d       t        t        g d
gg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       y )Nr   r   r0   r+   r  r  r   r   r   r   r   r   r   r  r  r  rO   rM   test_coverage_tie_handlingr!  2  s.   Ax3*>BAx3*>BAx3*>BAx3*>B	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGK	{5E4FGKrO   zy_true, y_scorec                 |    t        j                  t        d      5  t        | |       d d d        y # 1 sw Y   y xY w)Nz'Expected 2D array, got 1D array insteadr   )r   r   r   r   rT  s     rM   test_coverage_1d_error_messager#  B  s*     
z)S	Tvw' 
U	T	Tr  c            	      h   t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	gg dg      d       t        t        g d
gg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dg dgg dg dg      d       t        t        g dg dg d	gg dg dg dg      d       t        t        g dg dg d	gg dg dg dg      d       y )Nr   r0   r   r   r   r  r   r   r   r   r^   r   r   r  r  r  r  r  r  r  r@  r  r"   r   r  rO   rM   test_label_ranking_lossr&  Q  s   *QF8tTl^DaH*QF8tTl^DaH*I;9J8KLaP*I;9J8KLeT*I;9J8KLaP*I;9J8KLeT*I;9J8KLeT*I;9J8KLeT *QF8tTl^DaH*QF8tTl^DaH*QF8sCj\BAF*QF8sCj\BAF*I;9J8KLaP*I;9J8KLaP*I;9I8JKQO*I;9I8JKQO Iy1OY3OP
 	9-y)/T	
 	"	 	9-y)/T	
 	"	rO   csr_containerc                 z    t        t         | t        j                  g dg dg            g dg dg      d       y )Nr   r   r  r  r   )r"   r   r5   r   )r'  s    rM   test_label_ranking_loss_sparser)  |  s4    "((Iy#9:;mY=W	
 		rO   c                  .   t        j                  t              5  t        ddgddggddg       d d d        t        j                  t              5  t        ddgddggddgg       d d d        t        j                  t              5  t        ddgddggdgdgg       d d d        t        j                  t              5  t        ddggddgddgg       d d d        t        j                  t              5  t        dgdggddgddgg       d d d        t        j                  t              5  t        ddgddggdgdgg       d d d        y # 1 sw Y   !xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   sxY w# 1 sw Y   y xY w)Nr   r0   )r   r   r   r   r  rO   rM   $test_ranking_appropriate_input_shaper+    sI   	z	"QFQF+aV4 
#	z	"QFQF+q!fX6 
#	z	"QFQF+qcA3Z8 
#	z	"QF8q!fq!f%56 
#	z	"QC!:AA'78 
#	z	"QFQF+qcA3Z8 
#	" 
#	"	"	"	"	"	"	"	"	"	"	"sG   EEE'?E36E?.FEE$'E03E<?FFc                     t        t        ddggddgg      d       t        t        ddggddgg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g dgg dg      d       t        t        g d	gg dg      d       t        t        g d
gg dg      d       y )Nr0   r   r   r   r  r   r   r   r   r   r%  r  rO   rM   test_ranking_loss_ties_handlingr-    s    *QF8sCj\BAF*QF8sCj\BAF*I;9I8JKUS*I;9I8JKUS*I;9I8JKQO*I;9I8JKQO*I;9I8JKQO*I;9I8JKQOrO   c                      t        dd      \  } }| dz   }t        ||       t        j                  j	                  d      j                  d      \  }}t        ||       y Nr   rM  )r/   r  r0   )r+   r   rM  )r   _test_dcg_score_forr5   r9   r:   random_samplery   rL   rJ   s      rM   test_dcg_scorer3    sT    .ALIAvgkG(ii++A.<<\JOFG(rO   c                     t        j                  t        j                  | j                  d         dz         }t	        | |       }t	        | |      }||k  j                         sJ t	        | | d      |k  j                         sJ |j                  | j                  d   fk(  sJ |j                  | j                  d   fk(  sJ |t        j                  t        j                  |       d d d d df   |z  j                  d            k(  sJ y )Nr0   r+   r   kr   rQ   axis)
r5   log2r6   r4   r   r_  r   r*  sortrT   )rL   rJ   discountidealrc   s        rM   r0  r0    s    wwryya1A56Hvv.Evw/EUN!!!vv3u<AACCC;;6<<?,,,,;;6<<?,,,,FMM2776?1dd7#;h#F"K"KQR"K"STTTTrO   c            	         t        j                  t        j                  d      g      } t        j                  | j                        }t        | |      }t        | |d      }dt        j                  t        j                  dd            z  }|t        j                  |j                         | j                         z  g      k(  sJ |t        j                  || d d d d df   z  j                         g      k(  sJ d|dd	d f<   t        | |      }t        | |d      }|t        j                  || d d d d df   z  j                         g      k(  sJ |t        j                  |d d j                         | dd	d f   j                         z  |dd  j                         | dd d	f   j                         z  z   g      k(  sJ y )
Nr   Tignore_tiesr0   r+   r   rQ   r   r   )r5   r6  r6   r   r4   r   r9  r   r*  rT   r  )rL   rJ   dcgdcg_ignore_ties	discountss        rM   test_dcg_tiesrC    s   ZZ1'Fhhv||$G
VW
-C(dKOBGGBIIaO,,I&--6;;=!@ ABBBBfmmi&DbD/.I-N-N-P,QRRRRGAqrEN
VW
-C(dKOfmmi&DbD/.I-N-N-P,QRRRR&--bqM&AB-"4"4"66m!F1bqb5M$6$6$889	
   rO   c            	          t        j                  d      j                  d      } t        | | dd      t	        j
                  t        | | dd            k(  sJ y )N   )r+   r   r   T)r6  r?  )r5   r6   rS   r   r   r*  )r   s    rM   test_ndcg_ignore_ties_with_krF    sP    
		"f%AaaT2fmm1a1$/7   rO   c                      t        j                  g dg      } t        j                  g dg      }d}t        j                  t        |      5  t        | |       ddd       y# 1 sw Y   yxY w)zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{Gzg(\gGz޿g(\?gQ?)gQ?gףp=
?r   gQ?gHzG?z7ndcg_score should not be used on negative y_true valuesr   N)r5   r   r   r   r   r   )rL   rJ   r   s      rM    test_ndcg_negative_ndarray_errorrH    sQ    XX89:Fhh678GP	z)9	:67# 
;	:	:s   A$$A-c                     t        j                  d      j                  dd      } | t         j                  j	                  d      j                  dd| j                        z   }t        | |      }t        | |d	      }|t        j                  |      k(  sJ |t        j                  d
      k(  sJ |dz  }t        | |      t        j                  d
      k(  sJ y )NF   r   rM  r   皙ɿr   r   Tr>  r^   r1  )
r5   r6   rS   r9   r:   rd  r4   r   r   r*  )rL   rJ   ndcgndcg_no_tiess       rM   test_ndcg_invariantrN    s    YYr]""1b)Fryy,,Q/77c7UUGfg&Dfg4@L6==....6==%%%%tOGfg&&--*<<<<rO   r?  c           
      ,   dt        j                  d      d d z  }t        j                  t        j                  ddd      d      }|t         j                  j                  d      j                  dd	|j                  
      z   }t        |||       t        j                  dt        j                  t        j                  dd            z        k(  sJ t        |||       t        j                  dt        j                  t        j                  dd            z        k(  sJ t        |||       t        j                  dt        j                  t        j                  dd            z        k(  sJ t        ||d|       t        j                  dt        j                  t        j                  dd            z        k(  sJ t        |||       t        j                  dt        j                  t        j                  dd            z  j                               k(  sJ t!        |||       t        j                  dt        j                  t        j                  dd            z  j                               k(  sJ dt        j"                  d      z  }dt        j                  t        j                  dd            z  j%                         }t        |||       t        j                  |t        j"                  d      z        k(  sJ t        |||       t        j                  t        j"                  d            k(  sJ t!        |||       t        j                  |      k(  sJ t        |||       t        j                  d      k(  sJ y )Nr   r   r   r   rQ   )r   r0   r   rK  r   r   r>  r+   r0   rM  )log_baser?  )r   r   	   r^   )r5   eyetiler6   r9   r:   rd  r4   r   r   r*  r9  r   log10r   r  r   r   rT   )r?  rL   rJ   y_score_noisyexpected_dcg_scores        rM   test_ndcg_toy_examplesrW    s   2AFggbii2r*F3Gbii33A6>>c ?  M [	q277299Q?33	45 5 5 ;	q277299Q?33	45 5 5 [	q277299Q?33	45 5 5 "+	q288BIIaO44	56 6 6 fg;?6==	
RWWRYYq!_%	%++-D    VW+>&--	
RWWRYYq!_%	%++-C     Fbggbii1o66;;=[	)BGGAJ6	78 8 8 [	rwwqz	"# # # VW+>&--C    fg;?6==QTCUUUUrO   c                      d} t        j                  t        |       5  t        dggdgg       ddd       y# 1 sw Y   yxY w)zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r   r0   N)r   r   r   r   )r  s    rM   test_ndcg_error_single_documentrY    s:    	  
z	1QC5A3%  
2	1	1s	   8Ac                      t        dd      \  } }| dz   }t        ||       t        j                  j	                  d      j                  d      \  }}t        ||       y r/  )r   _test_ndcg_score_forr5   r9   r:   r1  r2  s      rM   test_ndcg_scorer\    sT    .ALIAvgkG)ii++A.<<\JOFG)rO   c                    t        | |       }t        | |      }||k  j                         sJ | dk(  j                  d      }||    t        j                  t	        j
                  | j                                     k(  sJ ||   t        j                  t	        j                  |j                                     k(  sJ ||    t        j                  t        | |      |    t        | |       |    z        k(  sJ ||   t        j                  t	        j                  |j                                     k(  sJ |j                  | j                  d   fk(  sJ |j                  | j                  d   fk(  sJ y )Nr   r0   r7  )
r   r_  r   r*  r5   r   rT   r   r   r4   )rL   rJ   r<  rc   all_zeros        rM   r[  r[    sR   /E0EUN!!!!  a (H(v}}RWWxi__5F-GHHHH?fmmBHHX\\^,DEEEE(v}}67+XI6
VV
,hY
7	8     ?fmmBHHX\\^,DEEEE;;6<<?,,,,;;6<<?,,,,rO   c            	      .   t        j                  g d      } t        | | d      dk(  sJ t        | | d      dk(  sJ t        j                  t
              5  t        | | d      sJ 	 d d d        t        j                  t
              5  t        | | d      sJ 	 d d d        t        j                  t
              5  t        | | d      sJ 	 d d d        t        j                  g d      }t        | |d      }t        | |      }||k(  sJ t        | |d	      d
k(  sJ t        d      \  } }}t        j                  ddd      D ]%  }t        t        | ||      t        | ||             ' y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nr  r0   )rv   gMbP?gg?r   )r   r   r   g{Gz?r   r   Tr   g-C6?r   )
r5   r   r   r   r   r   rN   linspacer"   r   )rL   r  roc_auc_with_max_fpr_oneunconstrained_roc_aucrK   ry   rv   s          rM   test_partial_roc_auc_scorerc  .  sY   XXl#F3q88871<<<	z	"VVT::: 
#	z	"VVS999 
#	z	"VVQ777 
# xx+,H,VXqI)&(;#'<<<<373>>>'t4FFA;;tQ*&&':"667;	
 + 
#	"	"	"	"	"s$   E2
E?=F2E<?FFzy_true, k, true_scorer   r+   r   r   c                     t        j                  g dg dg dg dg      }t        | ||      }|t        j                  |      k(  sJ y )Nr  r   r   r   r   r   r   r   r   r   r   r   r   r5  r5   r   r   r   r*  )rL   r6  
true_scorerJ   rc   s        rM   test_top_k_accuracy_scorerj  H  sJ     hh    		
G !A6EFMM*----rO   zy_score, k, true_score)rQ   rQ   r0   r0   )rQ   r0   rQ   r0   )r   r   r   r   )r   r   r   r   c                 ,   g d}| j                         dk\  r| j                         dk  rdnd}|dk(  r"| |kD  j                  t        j                        n|}t        || |      }t        ||      }||cxk(  rt        j                  |      k(  sJ  J y )Nr  r   r0   r   r5  )	minmaxastyper5   int64r   r   r   r*  )rJ   r6  ri  rL   rm   rK   rc   	score_accs           rM    test_top_k_accuracy_score_binaryrq  ]  s     F{{})gkkmq.@aI78Avg	!))"((36F A6Evv.II:z!::::::rO   zy_true, true_score, labels)r   r0   r0   r+   r  )r   rA  rA  r   )r   r   r  rA  labels_as_ndarrayc                     |rt        j                  |      }t        j                  g dg dg dg dg      }t        | |d|      }|t	        j
                  |      k(  sJ y)z,Test when labels and y_score are multiclass.r  re  rf  rg  r+   r6  r   N)r5   r6  r   r   r   r*  )rL   ri  r   rr  rJ   rc   s         rM   0test_top_k_accuracy_score_multiclass_with_labelsru  t  s]     F#hh    		
G !AfEEFMM*----rO   c            
         t        j                  dddd      \  } }t        | |d      \  }}}}t        d      }|j	                  ||       t        ||f||f      D ]f  \  } }t        dd      D cg c]  }t        ||j                  |       |      ! }}t        j                  t        j                  |      dkD        rfJ  y c c}w )NrM  r1  r   )r  rD   n_informativer/   r.   r+   r5  )r   make_classificationr   r   r=   rq  r`   r   r>   r5   r_  r   )	rB   rC   X_trainX_testy_trainy_testrI   r6  scoress	            rM   $test_top_k_accuracy_score_increasingr~    s    ''BQDAq (811'M$GVWf
!
,CGGGWWf%'891HMaQS
HT1 C$5$5a$8A> 	 
 vvbggfo)*** :
s   2$Cc                     t        j                  g dg dg dg dg      }t        | ||      t        j                  |      k(  sJ y )N)r   r   r   r   )r0   r   r   r   )r   r   r   r   r  r5  rh  )rL   r6  ri  rJ   s       rM   test_top_k_accuracy_score_tiesr    sE     hh		
G  15z9RRRRrO   z	y_true, kr   c                     t        j                  g dg dg dg dg      }d}t        j                  t        |      5  t        | ||      }d d d        dk(  sJ y # 1 sw Y   xY w)	Nr  )r   r   r   r   )r   r   r   r   )r   r   r   r   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r   r5  r0   )r5   r   r   r   r   r   )rL   r6  rJ   r   rc   s        rM   !test_top_k_accuracy_score_warningr    si     hh    		
G	6  
,4D	E$VW: 
FA:: 
F	Es   AA'zy_true, y_score, labels, msg)r   g=
ףp=?r0   r+   )r   r   r   )r   r   r   r   )r   r   r   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)r   r   r   r   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r0   r+   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).r  z3'y_true' contains labels not in parameter 'labels'.)r   r   r   )r   r   r   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                     t        j                  t        |      5  t        | |d|       d d d        y # 1 sw Y   y xY w)Nr   r+   rt  )r   r   r   r   )rL   rJ   r   rC  s       rM   test_top_k_accuracy_score_errorr    s-    f 
z	-VW&A 
.	-	-s   5>c                      | g dg dg      }t        j                  g dg dg      }t        ||      }|t        j                  d      k(  sJ y )Nr   r   )r   r   r   r  )r5   r   r   r   r*  )r'  rL   rJ   results       rM   Otest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_inputr  )	  sI     Iy12Fhh34G267CFV]]5))))rO   metricr3  ))FT)r   r0   )r   r^   )zeroonec                    t         j                  j                  d      }d|d   }}|j                  ||d      }|j	                  |      } | |||      }t        |t              rt        j                  |      rJ y|\  }}	}
t        j                  |      j                         rJ t        j                  |	      j                         rJ t        j                  |
      j                         rJ y)zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   rM  rQ   T)r   replacer  N)	r5   r9   r:   choicerH  
isinstancerU   r   any)r  r3  rG   rD   rW   rL   y_probar  metric_1metric_2r   s              rM   #test_ranking_metric_pos_label_typesr  5	  s     ))


#Cwr{yIZZiZ>Fhhy!GFGy9F&% 88F####)/&(J88H%))+++88H%))+++88J'++----rO   c                     t         j                  j                  |       }|j                  ddd      }|j	                  d      }t        ||      \  }}}t        j                  |d         sJ y)zCheck that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r+   rM  r   N)r5   r9   r:   r   rH  r   isinf)r9  rG   rL   rJ   ry   r   s         rM   )test_roc_curve_with_probability_estimatesr  O	  sc     ))

 2
3C[[AB['FhhrlG 1Aq*88JqM"""rO   c                     |t        j                  |      }t        j                  ||z        }t        j                  | |z        }||fS )z0Dummy metric that returns a tuple of two values.)r5   	ones_likerT   )rL   rK   r   
y_pred_sum
y_true_sums        rM   _dummy_metricr  ]	  sG    V,./J./J
##rO   metric_funcr   r0   r+   r0   r   r+   c           
         t        j                  g d      }t        j                  g d      }t        ||| |      \  }}g }|D ]4  }||k\  j                  t              }|j                   | |||             6 t        |      t        |      k(  sJ t        j                  t        j                  |      dk        sJ |?t        |t        j                  t        j                  ||dk7                 ddd          n8t        |t        j                  t        j                  |            ddd          t        ||       y)z,Test `metric_at_thresholds` outputs correct.r   r   r   r   r   r   r   r   NrQ   )r5   r   r   rn  r8   rr   rV   r_  r   r!   r:  rR   )	r  r   rL   rJ   metric_valuesr   expected_scoresrm   rK   s	            rM   test_metric_at_thresholdsr  f	  s    XXo&Fhh01G 4M!M:
 O	Y&..s3{66WX   }Z00066"''*%*+++ 		'-12D*E FG"M	
 	
BGGBIIg,>$?"$EFM?3rO   c                 V    t        j                  |      t        j                  |       fS )z2Dummy metric that does not accept `sample_weight`.)r5   rT   )rL   rK   s     rM   _dummy_metric_no_sample_weightr  	  s    FF6NBFF6N++rO   c                     t        j                  g d      } t        j                  g d      }t        j                  g d      }t        j                  t        d      5  t        | |t        |      \  }}ddd       y# 1 sw Y   yxY w)zLTest `TypeError` is raised when `metric_func` does not take `sample_weight`.r   r   r   r   r   r   r0   r+   r   r0   r+   z"got an unexpected keyword argumentr   r   N)r5   r   r   r   r|  r   r  )rL   rJ   r   ry   s       rM   -test_metric_at_thresholds_sample_weight_errorr  	  sa    XXo&Fhh12GHH_-M	y(L	M#G;=
1 
N	M	Ms   !BB
	normalizec                 |   t        j                  g d      }t        j                  g d      }t        ||t        d| i      \  }}g }|D ]7  }||k\  j	                  t
              }|j                  t        |||              9 t        ||       t        |      t        t        j                  |            k(  sJ y)z@Test `metric_params` passed correctly to `metric_at_thresholds`.r   r  r  metric_params)r  N)
r5   r   r   r   rn  r8   rr   r!   rV   rR   )r  rL   rJ   r  r   expected_valuesrm   rK   s           rM   'test_metric_at_thresholds_metric_paramsr  	  s     XXo&Fhh12G 4Y7O!M: O	Y&..s3~ff	RS   M?3z?c"))G"45555rO   rW   c           	      (   t        j                  g d      }t        j                  g d      }t        ||t        | dd      \  }}g }|D ]8  }||k\  j	                  t
              }|j                  t        ||| d             : t        ||       y)z?Test `pos_label` is passed correctly to `metric_at_thresholds`.r   r  r   )rW   zero_divisionr  N)r5   r   r   r   rn  r8   rr   r!   )rW   rL   rJ   r  r   r  rm   rK   s           rM   #test_metric_at_thresholds_pos_labelr  	  s     XXo&Fhh01G 4$-B	!M: O	Y&..s3FFiqQ	
   M?3rO   c                  ~   t        j                  g d      } t        j                  g d      }t        j                  g d      }t        d      }|j                  t	        |            }| |   }||   }||   }t        | |t        |      \  }}	t        ||t        |      \  }
}t        ||
       t        |	|       y)z<Test `y_score` order does not effect `metric_at_thresholds`.r   )r   r   r   r   r   r  r  r   N)r5   r   r)   permutationrV   r   r   r!   )rL   rJ   r   rG   perm_indicesy_true_permy_score_permsample_weight_permr  thresh_1r  thresh_2s               rM   'test_metric_at_thresholds_y_score_orderr  	  s    XXo&Fhh01GHH_-M
R
 C??3w<0L&K<(L&|4-}Hh .\>ASHh Hh'Hh'rO   c                  N   t        j                  g d      } t        j                  g d      }t        j                  g d      }t        || t              \  }}t        || t              \  }}t	        ||       t	        ||       t        j                  g d      }t        j                  g d      }t        || t        |      \  }}t        || t        |      \  }}t	        ||       t        j                  t              5  t	        ||       ddd       y# 1 sw Y   yxY w)a  Test duplicate `y_score` edge cases in `metric_at_thresholds`.

    If there are duplicate `y_score` values and `y_true` differs between
    these duplicate values, `y_score` order will not affect metric output.

    However, if there are duplicate `y_score` values and `sample_weight` differs
    between these duplicate values, `y_score` order can affect metric output,
    as stable sort preserves relative order.
    )r   r   r   r   r   )r0   r   r0   r0   r   r   r  )r+   r+   r0   r   r0   r   N)r5   r   r   r   r!   r   r   AssertionError)	rJ   y_true_1y_true_2r  r  r  r  sample_weight_1sample_weight_2s	            rM   9test_metric_at_thresholds_y_score_order_duplicate_y_scorer  	  s     hh01Gxx(Hxx(H-hPHh-hPHhHh'Hh' hh/Ohh/O-'>Hh .'>Hh
 Hh'	~	&(+ 
'	&	&s   DD$c                  p   t        j                  g d      } t        j                  g d      }t        | |      \  }}}}}t        | |t        dddgi      \  }}t        ||       t        |ddddf   |       t        |ddddf   |       t        |ddddf   |       t        |ddddf   |       y)zTest `metric_at_thresholds` consistency with `confusion_matrix_at_thresholds`.

    This also checks output when `metric_func` returns a tuple of arrays.
    r   )r   r   r   r   r   r   r   r0   r  N)r5   r   r   r   r   r$   )	rL   rJ   rf  rg  rh  ri  thresholds_cmr  r   s	            rM   ;test_metric_at_thresholds_consistency_with_confusion_matrixr  
  s    
 XXo&Fhh01G(F)%Cc3
 !5)(QF9K!M: z=1 }Q1W-s3}Q1W-s3}Q1W-s3}Q1W-s3rO   c                      t        j                  g d      } t        j                  g d      }t        | |t        dt         j                  i      \  }}t        j
                  t        j                  |            sJ y)z,Test `metric_at_thresholds` with NaN output.)r   r   r   r   r   r  r  r  N)r5   r   r   r   r   r_  r   )rL   rJ   r  ry   s       rM   *test_metric_at_thresholds_with_nan_outputsr  "
  s_     XXo&Fhh01G+orvv5NM1 66"((=)***rO   c                      t        j                  g d      } t        j                  g d      }t        j                  t        d      5  t        | |d        d d d        y # 1 sw Y   y xY w)Nro  )r   r   r   r   z"Pass pos_label=None as keyword argr   )r5   r   r   r   FutureWarningr   rT  s     rM   ?test_confusion_matrix_at_thresholds_positional_args_deprecationr  0
  sG    XXl#Fhh+,G	m+O	P&vw= 
Q	P	Ps   
A!!A*)NF)r   rG  r   r  )r   r   numpyr5   r   scipyr   sklearnr   sklearn.datasetsr   sklearn.exceptionsr   sklearn.linear_modelr   sklearn.metricsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   sklearn.metrics._rankingr   r   r   sklearn.model_selectionr   sklearn.preprocessingr   sklearn.random_projectionr   sklearn.utils._testingr    r!   r"   r#   r$   sklearn.utils.extmathr%   sklearn.utils.fixesr&   sklearn.utils.validationr'   r(   r)   CURVE_FUNCSrN   r\   rg   ro   r   markparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r&  r.  r?  rD  rK  rQ  rU  r]  rb  rj  rm  rw  filterwarningsr}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r`   r  r  r  r!  r#  r&  r)  r+  r-  r3  r0  rC  rF  rH  rN  rW  rY  r\  r[  rc  rj  rq  ru  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rO   rM   <module>r     s    	     ; 5 3    ( 
 5 0 ;  * .  #	)#`2<&H8 $/
) 0
)
")&#))8)6oNdM	).&" 	,	+	,	&	/	t$	/X/Xd 	,	+	&	'9: 	,	&	t$	y!	/	D D:( 	,hbhh|.DiP/BHH)*	
 BHH\"	
 1BHH)*	
 BHH\"F	
 BHH)*#J	
 BHH\"	
 BHH)* 	
 ABHH)*	
 ABHH)*	
 ABHH\"	
}CFN 7P 8OFPP ' "%8	
' "%8	
F "B7	
 "c2	
 
4R8="F	0G"F	0&*L"
D686& {3# 4# {3E 4E0 LM(>	'JK(9:' ; L N' {3	& 4	& $/  0 >), $/` 0`F6D<=
=>*.$2 .	K!qc*	NQC!-	NQC!-	%sQC0	K	:	NM9=	NM9=	%}i@	K#,	NQC!-	NQC!-	%sQC0	K>	NNLA	NNLA	%~|D	KM:	NI}=	NI}=	%y-@)2'32' N 
K!1<G	Ki>	N$4lEJ	NKDA	KM5A	KM4@	NI}eD	NI}dC'' .
Q#saVaV,
Q#saVaV,	$sAhA7	$sAhA7	$qcA3/	$i;	$i=	$i=	'' $EF2 G2 							 
Q?@	QH?@	K!FG	K!FG.(	

"">48Sv.14X6#N 9:3@ 3)	 "G!RS TC m4j1q2 3 2 5*4nL  	$%	%&'
&'(((V .9 :9 	P)U($= u6#V 7#VL!*- 
4 	q$	q#	q$
.
. 	.	!1a(	.	!1c*	.	!1a(	&	'A.	&	'C0	&	'A.
	;
	;  	,	|4	,	l3	,	l3	&	'/CD	 ,tUm<. =.&+& 	q$	q#	q!
S
S 	q	q$ " 	 G
	
 	 +	
 !	 !0
	
 !	 1
	
 	  	
 	 A
	
 Fo.W	
KNQdBeQdB
 .9* :* &	3I9U C..(#$ 	 4/1J*KL4 M	4:,
	
 tUm46 56$ q!f-4 .4,(.%,P44
+>rO   