
    Q3j"                    ^   d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z;m<Z< d dl=m>Z> d dl?m@ZA d dl?mBZBmCZC d dlDmEZE d dlFmGZGmHZHmIZImJZJmKZKmLZL d dlMmNZN d dlOmPZPmQZQ d dlRmSZS d
dZTd ZUe
j                  j                  ddd de	j                  g      d        ZYe
j                  j                  dd gd fd dgdfg d!dfg      d"        ZZd# Z[d$ Z\e
j                  j                  d%      d&        Z^e
j                  j                  d%      d'        Z_e
j                  j                  d%      d(        Z`d) Zae
j                  j                  d*g d+ e	j                  g d,g d-g d.g d/g      fg d0g d1fg      d2        Zce
j                  j                  d*g d3 e	j                  g d/g d/g d4g d4g d5g      fg d6g d7fg      d8        Zdd9 Zed: Zfd; Zgd< Zhe
j                  j                  d=eP      e
j                  j                  d>eQ      d?               Zid@ Zje
j                  j                  dAg dB      dC        ZkdD ZldE Zme
j                  j                  dF e	j                  g dG       e	j                  g dG      dHdIf e	j                  g dG       e	j                  g dJ      dHdKf e	j                  g dG       e	j                  g dL      dHdMf e	j                  g dJ       e	j                  g dG      dHdNfg      dO        Zne
j                  j                  dP e	j                  g dQ       e	j                  g dR      dHdSfg      dT        ZodU ZpdV Zqe
j                  j                  dWdXdYidZdYid[dYd\d]d^d\dYdYd\d]d_d\g      d`        Zre
j                  j                  dad]d]d\d]fe	j                  dYd\e	j                  fd_dYd\d_fe	j                  e	j                  d\e	j                  fe	j                  e	j                  fg      db        Zte
j                  j                  dad]d]d\d]fe	j                  dYd\dYfe	j                  dcd\dcfe	j                  e	j                  d\e	j                  fe	j                  e	j                  fg      dd        Zude Zv eLef      e
j                  j                  dgdgdhz  digdhz  z   djgdkz  ddigdfdjgdkz  djgdkz  ddfdgdhz  digdhz  z   dgdhz  djgdhz  z   ddigdfdgdhz  digdhz  z   dgdhz  djgdhz  z   ddigdlfg      e
j                  j                  dWdYe	j                  g      dm                      Zwdn Zxdo Zye
j                  j                  dd de	j                  g      e
j                  j                  dpd gd gfg      e
j                  j                  dqe' ee(dr      e1e2g      ds                      Zze
j                  j                  dpd gd gfg      e
j                  j                  dqe' ee(dr      e1e2g      dt               Z{du Z|dv Z}dw Z~dx Ze
j                  j                  dydzd{g      d|        Zd} Ze
j                  j                  d~g d      d        Zd Zd Zd Ze
j                  j                  dg dfdjdgdfgddg      d        Zd Ze
j                  j                  dg d      d        Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze
j                  j                  d%      d        Zd Zd Zd Zd Zd Zd Zd Ze
j                  j                  dddg      d        Ze
j                  j                  d%      d        Ze
j                  j                  d%      d        Ze
j                  j                  d%      e
j                  j                  dddde	j                  e	j                  fg      d               Ze
j                  j                  ddg      e
j                  j                  d~g d      e
j                  j                  dd de	j                  g      d                      Ze
j                  j                  d~g d      d        Ze
j                  j                  dd de	j                  g      d        Zd Zd Ze
j                  j                  dd de	j                  g      d        Ze
j                  j                  ddd de	j                  g      d        Ze
j                  j                  ddd de	j                  g      d        Ze
j                  j                  ddd de	j                  g      d        Zd Zd Zd Zd Ze
j                  j                  d ejV                  dgd gdgd gg      df ejV                  d gdgdigdgg      df ejV                  g dg dg dg      dfg      d        Zd ZdÄ ZdĄ Zdń ZdƄ ZdǄ ZdȄ ZdɄ Ze
j                  j                  de	jj                  e	jl                  e	jn                  g      dʄ        Ze
j                  j                  de	jj                  e	jl                  e	jn                  g      d˄        Ze
j                  j                  dg dg dfg ddd gd dgdd ggfg d!g d͢g dg d΢gfg      dτ        ZdЄ Zdф Zd҄ Zdӄ ZdԄ ZdՄ Zdք Zdׄ Ze
j                  j                  dg d٢g dڢfg dۢg dڢfg dڢg dۢfg      d܄        Ze
j                  j                  dqe+e' ee(dcr      e0e1e2e"g      e
j                  j                  dg dޢ      d߄               Ze
j                  j                  d e	j                  d dg       e	j                  dd g      dYf e	j                  d dg       e	j                  d dg      d]f e	j                  d dg       e	j                  d d g      dYf e	j                  d d g       e	j                  d d g      d]fg      d        Ze
j                  j                  e
j                  j                  d e-e'e	j                         e-e(die	j                         e-e1e	j                         e-e2e	j                        g      d               Zd Zd Zd Zd Zd Zd Zd Ze
j                  j                  dg dg dddfg dg dddfg dg dddfg dg dddfg dg dddfg ddcdcgdcdcgdcdcggddfg dg dgg dg dgdd fg dg dg dg dgd digdfg dg dg dg dgd gdfg	      d        Zϐd Ze
j                  j                  d eC             d	        Zy(      N)partial)chainpermutationsproduct)linalgsparse)hamming)	bernoulli)datasetssvm)config_context)CalibratedClassifierCV)make_multilabel_classification)UndefinedMetricWarning)accuracy_scoreaverage_precision_scorebalanced_accuracy_scorebrier_score_lossclass_likelihood_ratiosclassification_reportcohen_kappa_scoreconfusion_matrixf1_scorefbeta_scorehamming_loss
hinge_lossjaccard_scorelog_lossmake_scorermatthews_corrcoefmultilabel_confusion_matrixprecision_recall_fscore_supportprecision_scorerecall_scorezero_one_loss)_check_targetsd2_brier_scored2_log_loss_score)cross_val_score)LabelBinarizerlabel_binarize)DecisionTreeClassifierdevice)get_namespace)yield_namespace_device_dtype_combinations)MockDataFrame)_array_api_for_testsassert_allcloseassert_almost_equalassert_array_almost_equalassert_array_equalignore_warnings)_nanaverage)CSC_CONTAINERSCSR_CONTAINERS)check_random_stateFc                    | t        j                         } | j                  }| j                  }|r||dk     ||dk     }}|j                  \  }}t        j                  |      }t        d      }|j                  |       ||   ||   }}t        |dz        }t
        j                  j                  d      }t
        j                  ||j                  |d|z        f   }t        t        j                   dd      dd	
      }	|	j#                  |d| |d|       j%                  ||d       }
|r	|
dddf   }
|	j'                  ||d       }||d }|||
fS )zMake some classification predictions on a toy dataset using an SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    N   %   r      linear)kernelrandom_stateF   )ensemblecv   )r   	load_irisdatatargetshapenparanger;   shuffleintrandomRandomStatec_randnr   r   SVCfitpredict_probapredict)datasetbinaryXy	n_samples
n_featuresprnghalfclfy_pred_probay_predy_trues                V/DATA/.local/lib/python3.12/site-packages/sklearn/metrics/tests/test_classification.pymake_predictionre   H   sZ    $$&AAQx1q51GGIz
		)A
R
 CKKNQ41qAy1}D ))


"C
a9cJ&6778A !xa05QC 771Ud8QuX.<<QtuXFL $AqD)[[45"FtuXF6<''    c            
         t        j                         } t        | d      \  }}}ddddddd	d
ddddddddddddddddddd}t        ||t	        j
                  t        | j                              | j                  d      }|j                         |j                         k(  sJ |D ]u  }|dk(  r#t        ||   t              sJ ||   ||   k(  r)J ||   j                         ||   j                         k(  sJ ||   D ]  }t        ||   |   ||   |           w t        |d   d   t              sJ t        |d   d   t              sJ t        |d   d   t              sJ t        |d   d   t              sJ y ) NFrW   rX   g7Mo?gUUUUUU?ghQ?   )	precisionrecallf1-scoresupportUUUUUU?gc1Ƹ?g433333?   g)k??$I$I?   gCFQ?gc?gƢ?K   )rl   rj   rk   rm   g?gD~WG?g]3p?)setosa
versicolor	virginica	macro avgaccuracyzweighted avgT)labelstarget_namesoutput_dictrx   rt   rj   rw   rm   )r   rG   re   r   rK   rL   lenrz   keys
isinstancefloatr4   rN   )irisrc   rb   _expected_reportreportkeymetrics           rd   ,test_classification_report_dictionary_outputr   z   s   D'UCFFA
 -)*	
 -*+	
 -)+	
 ++'	
 '++(	
5 OD #yyT../0&&F ;;=O002222*fSk5111#;/#"6666#;##%)=)B)B)DDDD)#.#OC$8$@&+fBUV /  oh/<eDDDok2;?GGGoh/	:C@@@ok29=sCCCrf   zero_divisionwarnrF   c                 .   g dg d}}t        j                  d      5 }t        j                  dd       t        ||| d       | d	k(  r3t	        |      d
kD  sJ |D ]  }d}|t        |j                        v rJ  n|rJ d d d        y # 1 sw Y   y xY w)Nabc)r   r   dTrecordalwaysz.+Use `zero_division`)message)r   r{   r   rF   z7Use `zero_division` parameter to control this behavior.)warningscatch_warningsfilterwarningsr   r|   strr   )r   rc   rb   r   itemmsgs         rd   0test_classification_report_zero_division_warningr      s    $oFF		 	 	- 	2IJF-T	
 F"v;?"?Oc$,,////  : 
.	-	-s   AB:BBzlabels, show_micro_avgTr   rF   r=   c                 h    ddgddg}}t        ||| d      }|rd|v sJ d|vsJ yd|v sJ d|vsJ y)a3  Check the behaviour of passing `labels` as a superset or subset of the labels.
    WHen a superset, we expect to show the "accuracy" in the report while it should be
    the micro-averaging if this is a subset.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27927
    r   rF   T)ry   r{   z	micro avgrx   Nr   )ry   show_micro_avgrc   rb   r   s        rd   1test_classification_report_labels_subset_supersetr      s`     VaVFF"66&dSFf$$$'''V###&(((rf   c                     t        j                  g dg dg      } t        j                  g dg dg      }t        | |      dk(  sJ t        | |       dk(  sJ t        ||      dk(  sJ t        |t        j                  |            dk(  sJ t        | t        j                  |             dk(  sJ t        | t        j                  | j
                              dk(  sJ t        |t        j                  | j
                              dk(  sJ y )Nr   rF   rF   rF   r   rF   r   r   rF         ?rF   r   )rK   arrayr   logical_notzerosrJ   y1y2s     rd   .test_multilabel_accuracy_score_subset_accuracyr      s    	9i(	)B	9i(	)B"b!S((("b!Q&&&"b!Q&&&"bnnR01Q666"bnnR01Q666"bhhrxx01Q666"bhhrxx01Q666rf   c            	      @   t        d      \  } }}t        | |d       \  }}}}t        |ddgd       t        |ddgd       t        |d	d
gd       t        |ddg       i ddifD ]  }t	        j
                         5  t	        j                  d       t        | |fi |}t        |dd       t        | |fi |}	t        |	dd       t        | |fi |}
t        |
d
d       t        t        | |fddi|d|z  |	z  d|z  |	z   z  d       d d d         y # 1 sw Y   xY w)NTrX   averageg\(\?g333333?r=   g)\(?g(\?皙?gRQ?   r   rX   errorbeta      )re   r"   r5   r6   r   r   simplefilterr#   r$   r   r4   r   )rc   rb   r   r]   rfskwargspsrsfss              rd   %test_precision_recall_f1_score_binaryr      s7   't4FFA 1NJAq!Qa$q1a$q1a$q1q2r(#
 	8,-$$&!!'* :6:B%b$2ff77B%b$2&&3F3B%b$2FF==f=R"$r	B7 '& .&&s   <BDD	z1ignore::sklearn.exceptions.UndefinedMetricWarningc                     dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddgd      k(  sJ dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddg      k(  sJ dt        ddgddgt	        d            k(  sJ t        ddgddgt	        d            t        j                  t        ddgddgd            k(  sJ y )	N      ?rF   r   r           infg     j@)r#   r$   r   r   r   pytestapprox rf   rd   +test_precision_recall_f_binary_single_classr     s;   
 /1a&1a&1111,1v1v....(Aq6Aq6****+q!fq!f15555/2r(RH5555,Bx"b2222(B8b"X....+r2hRuU|DDDDBx"be=RHr2hS1B   rf   c                  >   g d} g d}t        | t        j                  d            }t        |t        j                  d            }| |f||fg}t        |      D ]  \  }\  } }t	        | |g dd       }t        g d|       t	        | |g dd      }t        t        j                  g d      |       d	D ]5  }|d
k(  r|dk(  rt        t	        | |g d|      t	        | |d |             7  dD ]  }t        j                  t              5  t	        ||t        j                  d      |       d d d        t        j                  t              5  t	        ||t        j                  dd      |       d d d         t        j                  g dg dg      } t        j                  g dg dg      }t        | |d
ddg      \  }}	}
}t        t        j                  ||	|
g      t        j                  g d             y # 1 sw Y   xY w# 1 sw Y   (xY w)N)rF   rC   rC   r=   )rF   rF   rC   r=   r   classes)r   rF   r=   rC   r   ry   r   )r   r   r   r   r   macro)microweightedsamplesr   r   )Nr   r   r      r   r   r   rF   r   r   rF   rF   rF   r   rF   r   ry   )      ?rF   竪?)r+   rK   rL   	enumerater$   r5   meanr4   r   raises
ValueErrorr   r"   )rc   rb   
y_true_bin
y_pred_binrH   iactualr   r]   r   r   r   s               rd   $test_precision_recall_f_extra_labelsr   )  s    FF		!=J		!=JVz:67D(FFff_dS!";VD ff_gV!"''*C"DfM 8G)#QVVOWUVVD'J 8  /( 7]]:&Z		!gV ']]:&JryyQ/? '& 7 XXy),-FXXy),-F0	1a&JAq!Q !Q+RXX6G-HI '&&&s   #H$HH	H	c                     g d} g d}t        | t        j                  d            }t        |t        j                  d            }| |f||fg}t        |      D ]  \  }\  } }t	        t
        | |ddg      }t	        t
        | |d       }t        dd	g |d 
             t        d |d
             t        d |d
             t        d |d
             dD ]  } ||
       ||
      k7  rJ   y )N)rF   rF   r=   rC   )rF   rC   rC   rC   r   r   rF   rC   ry   r   r   r   r   r   UUUUUU?r   r   )r   r   r   )r+   rK   rL   r   r   r$   r5   r4   )	rc   rb   r   r   rH   r   	recall_13
recall_allr   s	            rd   &test_precision_recall_f_ignored_labelsr   W  s     FF		!=J		!=JVz:67D(FFL&&!QH	\66$G
!3*i.EFOYw-GH3Yz5RSGYw%?@ 6GW-G1LLLL 6  /rf   c            	          t        j                  g dg dg dg dg dg dg      } t        j                  g dg dg dg d	g d
g dg      }d}t        j                  t        |      5  t        | |d       ddd       y# 1 sw Y   yxY w)z:Test multiclass-multiouptut for `average_precision_score`.)r=   r=   rF   rF   r=   r   r   rF   r=   rF   r=   r   rF   ffffff?皙?皙?皙?333333?r   r   r   r   r   r   r   )r   r   r   )r   r   r   z.multiclass-multioutput format is not supportedmatchr=   	pos_labelN)rK   r   r   r   r   r   )rc   y_scoreerr_msgs      rd   -test_average_precision_score_non_binary_classr   n  s{    XX	
	F hh	
	G ?G	z	11= 
2	1	1s   ,BBzy_true, y_scorer   r   rF   r=   r   r   r   r   )r   r   r   r   rF   rF   rF   rF   rF   rF   rF   )r   r   r   r   r   333333?r   rp   rp   rF   rF   c                 &    t        | |      dk(  sJ y)a(  
    Duplicate values with precision-recall require a different
    processing than when computing the AUC of a ROC, because the
    precision-recall curve is a decreasing curve
    The following situation corresponds to a perfect
    test statistic, the average_precision_score should be 1.
    rF   Nr   rc   r   s     rd   -test_average_precision_score_duplicate_valuesr     s    8 #673q888rf   )r=   r=   rF   rF   r   )r   r   r   )r   r   r   r   )r   r   r   c                 &    t        | |      dk7  sJ y )Nr   r   r   s     rd   (test_average_precision_score_tied_valuesr     s    : #673s:::rf   c                      d} t        j                  t        |       5  t        g dg ddd       d d d        y # 1 sw Y   y xY w)NzNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.r   r   rF   r=   r=   r=   r   r   r   )r   warnsUserWarningr"   r   s    rd   (test_precision_recall_f_unused_pos_labelr    s9    
	  
k	-'yAw	
 
.	-	-s	   ;Ac            	          t        d      \  } }}d } || |        || D cg c]  }t        |       c}|D cg c]  }t        |       c}       y c c}w c c}w )NTr   c                 <   t        | |      }t        |ddgddgg       |j                         \  }}}}||z  ||z  z
  }t        j                  ||z   ||z   z  ||z   z  ||z   z        }|dk(  rdn||z  }	t        | |      }
t        |
|	d       t        |
dd       y )	N   rC         r   r=   decimal=
ףp=?)r   r6   flattenrK   sqrtr    r5   )rc   rb   cmtpfpfntnnumdentrue_mccmccs              rd   testz*test_confusion_matrix_binary.<locals>.test  s    ff-2Q!R12BB2gRggrBw27+rBw727CDq1cCi/!#x;!#tQ7rf   re   r   rc   rb   r   r  rZ   s        rd   test_confusion_matrix_binaryr    sZ    't4FFA8 	&	!&Q#a&&	!F#;FqCFF#;<	!#;
   AA
c            	          t        d      \  } }}d } || |        || D cg c]  }t        |       c}|D cg c]  }t        |       c}       y c c}w c c}w )NTr   c                 P    t        | |      }t        |ddgddggddgddggg       y )Nr  r  rC   r
  r!   r6   )rc   rb   r  s      rd   r  z5test_multilabel_confusion_matrix_binary.<locals>.test  s7    (82"a1b' 2b!Wq"g4FGHrf   r  r  s        rd   'test_multilabel_confusion_matrix_binaryr#    s[    't4FFAI 	&	!&Q#a&&	!F#;FqCFF#;<	!#;r  c            	          t        d      \  } }}dd} || |        || D cg c]  }t        |       c}|D cg c]  }t        |       c}d       y c c}w c c}w )NFr   c           	      P   t        | |      }t        |ddgddggddgddggd	d
gddggg       |rg dng d}t        | ||      }t        |ddgddggd	d
gddggddgddggg       |rg dng d}t        | ||      }t        |ddgddggd	d
gddggddgddggddgddggg       y )N/   r   r      &   r      rC      r   r=      )021r   r=   rF   r   )r,  r-  r.  3)r   r=   rF   rC   rs   r   r"  )rc   rb   string_typer  ry   s        rd   r  z9test_multilabel_confusion_matrix_multiclass.<locals>.test  s   (82q'Ar7#r1gAw%72r(QG9LM	

 %0Y(G2q'Ar7#r2hB%8B7RG:LM	

 *5%,(Ga1b'"bAr7#a2q'"a1a&!		
rf   T)r1  )Fr  r  s        rd   +test_multilabel_confusion_matrix_multiclassr2    s\    'u5FFA
6 	&	!&Q#a&&	!F#;FqCFF#;N	!#;s
   AA
csc_containercsr_containerc                    t        j                  g dg dg dg      }t        j                  g dg dg dg      } ||      } ||      } | |      } | |      }t        j                  g d      }dd	gddggdd	gddggd	d
gdd	ggg}	|||g}
|||g}|
D ]!  }|D ]  }t        ||      }t        ||	        # t        ||d      }t        |dd	gddggddgd	dggd	dgd
d	ggg       t        ||d
d	g      }t        |d	d
gdd	ggdd	gddggg       t        ||d
d	gd      }t        |d	d	gddggddgd	d	ggd	dgdd	ggg       t        |||d      }t        |d
d	gd
d
ggddgd	dggd	dgdd	ggg       y )Nr   r   rF   r   rF   rF   r   r   r   r   )r=   rF   rC   rF   r   r=   T
samplewiser   )ry   r9  )sample_weightr9  rC   r   )rK   r   r!   r6   )r3  r4  rc   rb   
y_true_csr
y_pred_csr
y_true_csc
y_pred_cscr:  real_cmtruespreds
y_true_tmp
y_pred_tmpr  s                  rd   +test_multilabel_confusion_matrix_multilabelrD    s   
 XXy)Y78FXXy)Y78Fv&Jv&Jv&Jv&J HHY'MAA1a&1a&!1QFQF3CDGZ,EZ,E
J,ZDBr7+    
%VV	EBraVaV,1v1v.>!Q!Q@PQR 
%VVQF	CBraVaV,1v1v.>?@ 
%VVQFt	TBraVaV,1v1v.>!Q!Q@PQR 
%m
B raVaV,1v1v.>!Q!Q@PQRrf   c            	         t        j                  g dg dg dg      } t        j                  g dg dg dg      }t        j                  t        d      5  t        | |d	d
g       d d d        t        j                  t        d      5  t        | |g dg dg dg       d d d        d}t        j                  t        |      5  t        | |dg       d d d        d}t        j                  t        |      5  t        | |dg       d d d        t        j                  t        d      5  t        g dg dd       d d d        d}t        j                  t        |      5  t        g dg dgg dg dg       d d d        y # 1 sw Y   $xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w# 1 sw Y   y xY w)Nr   r6  r7  r   r   r   inconsistent numbers of samplesr   rF   r=   r:  z)Sample weights must be 1D array or scalarrF   r=   rC   )r=   rC   r   )rC   r   r   z%All labels must be in \[0, n labels\)r   r   rC   zSamplewise metricsr   r   Tr8  z'multiclass-multioutput is not supported)r=   rF   r   rF   r   r=   )rK   r   r   r   r   r!   )rc   rb   r   s      rd   'test_multilabel_confusion_matrix_errorsrJ  F  sT   XXy)Y78FXXy)Y78F 
z)J	K#FF1a&I 
L	z)T	U#F9i*K	
 
V 7G	z	1#FFB4@ 
26G	z	1#FFA3? 
2 
z)=	>#IyTJ 
? 8G	z	1#Y	$:Y	<RS 
2	1+ 
L	K	U	U 
2	1 
2	1 
?	>
 
2	1sH   FF	F$>F01F<)GFF!$F-0F9<GGz%normalize, cm_dtype, expected_results))truer   TUU?)predr   rL  )allr   geq?)Nr   r=   c                     g ddz  }t        t        t        g d             }t        |||       }t	        ||       |j
                  j                  |k(  sJ y )Nr   r   	normalize)listr   r   r   r3   dtypekind)rQ  cm_dtypeexpected_resultsy_testrb   r  s         rd   test_confusion_matrix_normalizerX  d  sP     ]F%i012F	&&I	>BB()88==H$$$rf   c                     g d} g d}t        | |d      }|j                         t        j                  d      k(  sJ t	        j
                         5  t	        j                  dt               t        | |d      }d d d        j                         t        j                  d      k(  sJ t	        j
                         5  t	        j                  dt               t        || d       d d d        y # 1 sw Y   wxY w# 1 sw Y   y xY w)	N)r   r   r   r   rF   rF   rF   rF   )r   r   r   r   r   r   r   r   rK  rP         @r   rM  r   )r   sumr   r   r   r   r   RuntimeWarning)rW  rb   cm_truecm_preds       rd   ,test_confusion_matrix_normalize_single_classr_  u  s    %F%Fvv@G;;=FMM#.... 
	 	 	"g~6"66VD 
# ;;=FMM#....		 	 	"g~66: 
#	" 
#	" 
#	"s   )C2 )C>2C;>Dc                      g d} g d}t        j                  t        d      5  t        ||        ddd       y# 1 sw Y   yxY w)z8Test `confusion_matrix` warns when only one label found.r   r   r   r   zA single label was found inr   N)r   r  r  r   )rW  rb   s     rd   "test_confusion_matrix_single_labelrb    s2    FF	k)F	G( 
H	G	Gs	   :Azparams, warn_msg)rF   rF   rF   r   r   r   rc   rb   z?`positive_likelihood_ratio` is ill-defined and set to `np.nan`.)r   r   r   r   r   r   zdNo samples were predicted for the positive class and `positive_likelihood_ratio` is set to `np.nan`.r   r   r   rF   rF   rF   z?`negative_likelihood_ratio` is ill-defined and set to `np.nan`.z9No samples of the positive class are present in `y_true`.c                 z    t        j                  t        |      5  t        di |  d d d        y # 1 sw Y   y xY wNr   r   )r   r  r  r   )paramswarn_msgs     rd   test_likelihood_ratios_warningsri    s*    X 
k	2)&) 
3	2	2   1:zparams, err_msg)r   rF   r   rF   r   rF   rF   r   r   r=   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                 z    t        j                  t        |      5  t        di |  d d d        y # 1 sw Y   y xY wrf  )r   r   r   r   )rg  r   s     rd   test_likelihood_ratios_errorsrm    s)    $ 
z	1)&) 
2	1	1rj  c                     t        j                  dgdz  dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z         }t        | |      \  }}t        |d       t        |d	       t        | |       \  }}t	        |t         j
                  dz         t        |t        j                  d      d
       t        j                  dgdz  dgdz  z         }t        | ||      \  }}t        |d       t        |d       y )NrF   rC   r   r  r=   
   r  g?g_B{	%?g-q=)rtolr      r   r   rG  gUUUUUU@gqq?)rK   r   r   r3   r6   nanr   )rc   rb   posnegr:  s        rd   test_likelihood_ratiosru    s     XXqcAgb()FXXqcAgb(A3723F&vv6HCC!C! 'vv6HCsBFFQJ'C!51
 HHcURZ3%!)34M&vv]SHCC C!rf   c                  h   t        j                  g d      } t        j                  g d      }t        | |d      \  }}|t        j                  d      k(  sJ t        j                  g d      } t        j                  g d      }t        | |d      \  }}|t        j                  d      k(  sJ y)zTest that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
    LR- when `replace_undefined_by=1` is set.r7  r   rF   replace_undefined_byr   r   N)rK   r   r   r   r   )rc   rb   positive_likelihood_ratior   negative_likelihood_ratios        rd   1test_likelihood_ratios_replace_undefined_by_worstr{    s    
 XXi FXXi F#:Q$ q %c(:::: XXi FXXi F#:Q$ A  %c(::::rf   rx  LR+r   LR-g      )r|  r}  r   rr  rZ  c                     t        j                  ddg      }t        j                  ddg      }d}t        j                  t        |      5  t        |||        ddd       y# 1 sw Y   yxY w)zTest that class_likelihood_ratios raises a `ValueError` if the input dict for
    `replace_undefined_by` is in the wrong format or contains impossible values.rF   r   zGThe dictionary passed as `replace_undefined_by` needs to be in the formr   rw  N)rK   r   r   r   r   r   )rx  rc   rb   r   s       rd   6test_likelihood_ratios_wrong_dict_replace_undefined_byr    sW     XXq!fFXXq!fF
SC	z	-F1E	
 
.	-	-s   A$$A-zreplace_undefined_by, expectedc                    t        j                  g d      }t        j                  g d      }t        |||       \  }}t        j                  |      rt        j                  |      sJ y|t	        j
                  |      k(  sJ y)zTest that the `replace_undefined_by` param returns the right value for the
    positive_likelihood_ratio as defined by the user.r7  r   rw  NrK   r   r   isnanr   r   )rx  expectedrc   rb   ry  r   s         rd   0test_likelihood_ratios_replace_undefined_by_0_fpr     sp     XXi FXXi F#:-A$ q 
xxxx1222(FMM(,CCCCrf   r   c                    t        j                  g d      }t        j                  g d      }t        |||       \  }}t        j                  |      rt        j                  |      sJ y|t	        j
                  |      k(  sJ y)zTest that the `replace_undefined_by` param returns the right value for the
    negative_likelihood_ratio as defined by the user.r   r   rw  Nr  )rx  r  rc   rb   r   rz  s         rd   0test_likelihood_ratios_replace_undefined_by_0_tnr  <  sp     XXi FXXi F#:-A$ A  
xxxx1222(FMM(,CCCCrf   c                     t        j                  dgdz  dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z   dgdz  z         }t        | |      }t        |dd	
       |t        ||       k(  sJ t        j                  | dgdz        } t        j                  |dgdz        }t        | |ddg      |k(  sJ t        t        | |       d       t        j                  dgdz  dgdz  z   dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z         }t        t        | |      dd
       t        j                  dgdz  dgdz  z   dgdz  z         } t        j                  dgdz  dgdz  z   dgdz  z         }t        t        | |      dd
       t        t        | |d      dd
       t        t        | |d      dd
       y )Nr   (   rF   <   rr   ro  2   gʡE?rC   r  r=   r   r   r   .   ,   4          g??g+?r@   weightsg_vO?	quadraticg#?)rK   r   r   r4   append)r   r   kappas      rd   test_cohen_kappar  X  s    
1#(aS2X%	&B	1#(aS2X%b0A38;	<Bb"%Eua0%b"---- 
2sQw	B	2sQw	BRQF3u<<<)"b137 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"b161E 
1#(aS2X%b0	1B	1#(aS2X%b0	1B)"b161E)"b(CVUVW"b+6rf   )category	test_caser   r=   rC   ro  r@   c                     | \  }}}}t        j                  |      t        j                  |      }}t        |||||      }t        ||d       y)aM  Test that cohen_kappa_score handles divisions by 0 correctly by returning the
    `replace_undefined_by` param. (The first test case covers the first possible
    location in the function for an occurrence of a division by zero, the last three
    test cases cover a zero division in the second possible location in the
    function.)ry   r  rx  T)	equal_nanN)rK   r   r   r3   )r  rx  r   r   ry   r  scores          rd   test_cohen_kappa_undefinedr  w  sS    4 (BFGXXb\288B<B

1E E/4@rf   c                     ddg} t        j                  dgdz  dgdz  z         }t        j                  dgdz        }t        j                  t        d      5  t        |||        d	d	d	       ddg} t        j                  dgdz  dgdz  z         }t        j                  dgdz  dgdz  z         }t        j                  t        d
      5  t        |||        d	d	d	       y	# 1 sw Y   xY w# 1 sw Y   y	xY w)zZTest that cohen_kappa_score raises UndefinedMetricWarning when a division by 0
    occurs.rF   r=   r   rC   ro  zC`y2` contains no labels that are present in both `y1` and `labels`.r   r   Nz6`y1`, `y2` and `labels` have only one label in common.)rK   r   r   r  r   r   ry   r   r   s      rd   &test_cohen_kappa_zero_division_warningr    s     VF	1#'QC!G#	$B	1#(	B	S
 	"b0	
 VF	1#'QC!G#	$B	1#'QC!G#	$B	F
 	"b0	
 

 

 
s   C'C3'C03C<c                      ddg} t        j                  dgdz  dgdz  z         }t        j                  dgdz        }t        j                  t        d      5  t        ||| 	       d
d
d
       y
# 1 sw Y   y
xY w)zLTest that correct error is raised when users pass labels that are not in y1.rF   r=   r   r   r   ro  z6At least one label in `labels` must be present in `y1`r   r   N)rK   r   r   r   r   r   r  s      rd   (test_cohen_kappa_score_error_wrong_labelr    sn    VF	3%!)seai'	(B	3%"*	B	R
 	"b0
 
 
s   A11A:zy_true, y_predr   r   c                     t        j                         5  t        j                  d        | |||      }ddd       t        j                  |      rt        j                        sJ y|k(  sJ y# 1 sw Y   >xY w)zmCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
    No warnings should be raised.
    r   r   N)r   r   r   rK   r  )r   rc   rb   r   results        rd   !test_zero_division_nan_no_warningr    sh     
	 	 	"g&mD 
# 
xxxx&&& 
#	"s   !A33A<c                     t        j                  t              5   | ||d      }ddd       dk(  sJ y# 1 sw Y   xY w)ztCheck the behaviour of `zero_division` when setting to "warn".
    A `UndefinedMetricWarning` should be raised.
    r   r  Nr   )r   r  r   )r   rc   rb   r  s       rd   test_zero_division_nan_warningr    s9     
,	-f= 
.S== 
.	-s   6?c                     t         j                  j                  |       }|j                  ddd      }|j                  ddd      }t	        t        ||      t        j                  ||      d   d       y )Nr   r=   rr   sizer   rF   ro  )rK   rO   rP   randintr4   r    corrcoef)global_random_seedr^   rc   rb   s       rd   -test_matthews_corrcoef_against_numpy_corrcoefr    se    
))

 2
3C[[AB['F[[AB['F&&)2;;vv+Ft+Lbrf   c                    t         j                  j                  |       }|j                  ddd      }|j                  ddd      }|j	                  d      }t        |||      }t        |      }t        t        |      D 	cg c]A  }t        |      D ]1  }t        |      D ]!  }	|||f   |||	f   z  ||	|f   |||f   z  z
  # 3 C c}	}}      }
t        t        |      D cg c]c  }|d d |f   j                         t        j                  t        |      D cg c]  }t        |      D ]  }||k7  s	|||f    ! c}}      z  e c}}}      }t        j                  t        |      D cg c]c  }||d d f   j                         t        j                  t        |      D cg c]  }t        |      D ]  }||k7  s	|||f    ! c}}      z  e c}}}      }|
t        j                  ||z        z  }t        |||      }t        ||d       y c c}	}}w c c}}w c c}}}w c c}}w c c}}}w )Nr   r=   rr   r  rG  ro  )rK   rO   rP   r  randr   r|   r[  ranger  r    r4   )r  r^   rc   rb   r:  CNkmlcov_ytypr   gcov_ytytcov_ypyp
mcc_jurmanmcc_ourss                    rd   %test_matthews_corrcoef_against_jurmanr    s.    ))

 2
3C[[AB['F[[AB['FHHRLM}EAAA 1X	
1X1X adGa1g!Q$!AqD' 11  2 2	
H  1X	
  adGKKMffuQxLx!qAQ!Va1ggxLMN	
H vv 1X	
  adGKKMffuQxLx!qAQ!Va1ggxLMN	
H BGGHx$788J }MH*b11	
 M	
 M	
sC   AH )8H-!H':H'H-88H:0H4	H4H:'H-4H:c           	         t         j                  j                  |       }|j                  ddd      D cg c]  }|dk(  rdnd }}t	        t        ||      d       |D cg c]  }|dk(  rdnd }}t	        t        ||      d       t        |ddg	      }t        j                  |dd      }t	        t        ||      d       t	        t        g d
g d
      d       t	        t        |dgt        |      z        d       g d}g d}t	        t        ||      d       dgdz  dgdz  z   }t        j                  t              5  t	        t        |||      d       d d d        y c c}w c c}w # 1 sw Y   y xY w)Nr   r=   rr   r  r   r   r   r   r   ra  r   )rF   r   rF   rF   r   rF   rF   rF   r   rF   rF   rF   rF   rF   rF   rF   r   rF   rF   rF   )rF   rF   rF   r   r   rF   rF   rF   rF   r   rF   rF   rF   r   rF   rF   rF   r   rF   rF   rF   ro  rG  )rK   rO   rP   r  r4   r    r+   wherer|   r   r   AssertionError)	r  r^   r   rc   
y_true_invy_true_inv2y_1y_2masks	            rd   test_matthews_corrcoefr    sj   
))

 2
3C.1kk!QRk.HI.HQ!Vc$.HFI )&&93? 5;;Fqc#s*FJ;)&*=rB #s<K((;S1K)&+>C ),EsK )&3%#f+2EFL GC
FC)#s3S9 38qcBhD 
~	&-c3dKSQ 
'	&; J <. 
'	&s   E)"E.E33E<c                    t         j                  j                  |       }t        d      }d}|j	                  d|d      D cg c]  }t        ||z          }}t        t        ||      d       g d}g d}t        t        ||      d	       g d}g d
}t        t        ||      dt        j                  d      z         g d}g d}t        t        ||      d       g d}g d}t        t        ||      d       g d}	g d}
t        t        |	|
      d       g d}g d}g d}t        t        |||      d       g d}g d}g d}t        t        |||      d       y c c}w )Nr   r   r   rr   r  r   )r   r   rF   rF   r=   r=   )r=   r=   r   r   rF   rF   g      )rF   rF   r   r   r   r   ii  r   )rC   rC   rC   r   	r   rF   r=   r   rF   r=   r   rF   r=   )	rF   rF   rF   r=   r=   r=   r   r   r   )r   r   rF   rF   r=   rk  rF   rF   rF   rF   r   rG  r   r   rF   rF   r   r   )	rK   rO   rP   ordr  chrr4   r    r  )r  r^   ord_a	n_classesr   rc   
y_pred_bad
y_pred_minrb   r  r  r:  s               rd   !test_matthews_corrcoef_multiclassr  @  s_   
))

 2
3CHEI&)kk!YRk&HI&Hc%!)n&HFI )&&93?  F#J)&*=tD  F#J)&*=sRWWWEU?UV FF)&&93? FF)&&93? &C
%C)#s3S9 FF#M&&F FF M&&F_ Js   E%n_pointsd   i'  c                    t         j                  j                  |      d }fd}t        j                  ddg|       }t	        t        ||      d       t        j                  g d|       }t	        t        ||      d        ||       \  }}t	        t        ||      d       t	        t        ||       |||             y )Nc                     t        | |      }|d   }|d   }|d   }t        |       }||z   |z  }||z   |z  }||z  ||z  z
  }	||z  d|z
  z  d|z
  z  }
|	t        j                  |
      z  S )NrF   rF   )rF   r   r  rF   )r   r|   rK   r  )rc   rb   conf_matrixtrue_pos	false_pos	false_negr  pos_rateactivitymcc_numeratormcc_denominators              rd   mcc_safez1test_matthews_corrcoef_overflow.<locals>.mcc_safe}  s    &vv6t$%	%	v;y(H4y(H4 8+h.AA"X-X>!h,Orww777rf   c                 v    j                  |       }|dj                  |       dz
  z  z   }|dkD  }|dkD  }||fS )Nr   r   )random_sample)r  x_truex_predrc   rb   r^   s        rd   	random_ysz2test_matthews_corrcoef_overflow.<locals>.random_ys  sN    ""8,#!2!28!<s!BCC##v~rf   r   r   )r   r   rZ  )rK   rO   rP   repeatr4   r    )r  r  r  r  arrrc   rb   r^   s          @rd   test_matthews_corrcoef_overflowr  x  s     ))

 2
3C
8 ))S#J
)C)#s3S9
))OX
.C)#s3S9x(NFF)&&93?)&&98FF;STrf   c                  ,   t        d      \  } }}t        | |d       \  }}}}t        |g dd       t        |g dd       t        |g dd       t        |g d       t	        | |d	d
      }t        |dd       t        | |d
      }t        |dd       t        | |d
      }	t        |	dd       t	        | |d      }t        |dd       t        | |d      }t        |dd       t        | |d      }	t        |	dd       t	        | |d      }t        |dd       t        | |d      }t        |dd       t        | |d      }	t        |	dd       t        j                  t              5  t	        | |d       d d d        t        j                  t              5  t        | |d       d d d        t        j                  t              5  t        | |d       d d d        t        j                  t              5  t        | |dd       d d d        t        | |g dd       \  }}}}t        |g dd       t        |g dd       t        |g dd       t        |g d       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)NFr   r   )(\?Q?gzG?r=   )HzG?g
ףp=
?rp   )Q?333333?r  )ri   ro   rr   rF   r   r  g(\?r   r   gRQ?r   gGz?r   r   r   r   r/  r   )r  g=
ףp=?r  )r  rp   r   )r  r  r  )ri   rr   ro   )re   r"   r5   r6   r#   r$   r   r   r   r   r   )
rc   rb   r   r]   r   r   r   r   r   r   s
             rd   )test_precision_recall_f1_score_multiclassr    s.   'u5FFA 1NJAq!Qa!3Q7a!3Q7a!3Q7q,' 
1g	FBb$*	ffg	6Bb$*	&&'	2Bb$*		9Bb$*	ffg	6Bb$*	&&'	2Bb$*		<Bb$*	ffj	9Bb$*	&&*	5Bb$*	z	"	: 
#	z	"VVY7 
#	z	"3 
#	z	"FFIC@ 
# 1y$JAq!Q a!3Q7a!3Q7a!3Q7q,'! 
#	"	"	"	"	"	"	"s0   ,I&I2I><J
&I/2I;>J
Jr   )r   r   r   r   Nc                     t        j                  g dg      }t        j                  g dg      }t        ||g dg |       \  }}}}t        |d       t        |d       t        |d       | t        |g d       y y )Nr  r   r   rF   rF   )rC   r   rF   r=   )ry   warn_forr   r   r   rF   rF   r   )rK   r   r"   r6   )r   rc   rb   r]   r   r   r   s          rd   ;test_precision_refcall_f1_score_multilabel_unordered_labelsr    su     XX|n%FXX|n%F0|b'JAq!Q q!q!q!1l+ rf   c                  @   t        j                  g d      } t        j                  g d      }t        | |d       \  }}}}t        | |d      \  }}}}|t        j                  |      k(  sJ |t        j                  |      k(  sJ |t        j                  |      k(  sJ t        | |d      \  }}}}t        j                  |       }	|t        j
                  ||	      k(  sJ |t        j
                  ||	      k(  sJ |t        j
                  ||	      k(  sJ y )N)r   rF   r   r   rF   rF   r   rF   r   r   rF   r   rF   r   rF   )rF   rF   r   rF   r   rF   rF   rF   rF   r   rF   r   rF   r   rF   r   r   r   r  )rK   r   r"   r   bincountr   )
rc   rb   r   r   r   r   r]   r   r   rm   s
             rd   .test_precision_recall_f1_score_binary_averagedr    s    XXCDFXXCDF 4FFDQMBB0QJAq!Q0TJAq!Qkk&!G

2w////

2w////

2w////rf   c                     t        j                  d      } 	 t        j                  g d      }t        j                  g d      }t        t	        ||d      dd       t        t        ||d      dd       t        t        ||d      dd       t        j                  d	i |  y # t        j                  d	i |  w xY w)
Nraise)rN  )r   rF   r=   r   rF   r=   )r=   r   rF   rF   r=   r   r   r   r   r=   r   )rK   seterrr   r4   r#   r$   r   )old_error_settingsrc   rb   s      rd   test_zero_precision_recallr    s     w/	(,-,-OFFGLcSTULI3PQRHVVWEsAN 			'&'		'&'s   A9B' 'B>c                      t        d      \  } }}t        | |ddg      }t        |ddgddgg       t        | |d	dg      }t        |d
d	gddgg       t        j                  |       dz   }t        | |d	|g      }t        |d
dgddgg       y )NFr   r   rF   r   r'  r   rC   r=   r+  ri   )re   r   r6   rK   max)rc   rb   r   r  extra_labels        rd   .test_confusion_matrix_multiclass_subset_labelsr     s    'u5FFA 
&&!Q	8BrRGaV,- 
&&!Q	8BrRGb!W-. &&.1$K	&&![1A	BBrRGaV,-rf   zlabels, err_msgz+'labels' should contain at least one label.r   z.At least one label specified must be in y_truez
empty listzunknown labels)idsc                     t        d      \  }}}t        j                  t        |      5  t	        |||        d d d        y # 1 sw Y   y xY w)NFr   r   r   )re   r   r   r   r   )ry   r   rc   rb   r   s        rd   test_confusion_matrix_errorr    s;     (u5FFA	z	17 
2	1	1s   AAc            	         g d} t        j                  t        |             }t        | |       }|j                  t         j
                  k(  sJ t         j                  t         j                  t         j                  fD ]@  }t        | | |j                  |d            }|j                  t         j
                  k(  r@J  t         j                  t         j                  d t        fD ]@  }t        | | |j                  |d            }|j                  t         j                  k(  r@J  t        j                  t        |       dt         j                        }t        | | |      }|d   dk(  sJ |d   d	k(  sJ t        j                  t        |       d
t         j
                        }t        | | |      }|d   d
k(  sJ |d   dk(  sJ y )Nr   F)copyrG  l    rS  r   r   r  l    l    )rK   onesr|   r   rS  int64bool_int32uint64astypefloat32float64objectfulluint32)rZ   weightr  rS  s       rd   test_confusion_matrix_dtyper  &  sr   AWWSV_F	!Q	B88rxx((BHHbii0a&--E-2RSxx288### 1 **bjj$7a&--E-2RSxx2::%%% 8
 WWSVZryy9F	!Qf	5Bd8z!!!d8z!!! WWSV0AF	!Qf	5Bd8****d8r>>rf   rS  )Int64Float64booleanc                     t        j                  d      }t        j                  g d      }|j	                  ||       }|j	                  g dd      }t        ||      }t        ||      }t        ||       y)zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    pandas)	rF   r   r   rF   r   rF   rF   r   rF   r  )	r   r   rF   rF   r   rF   rF   rF   rF   r
  N)r   importorskiprK   r   Seriesr   r6   )rS  pd	y_ndarrayrc   y_predictedoutputexpected_outputs          rd   %test_confusion_matrix_pandas_nullabler"  A  sj     
		X	&B45IYYyY.F))7w)GKfk2F&y+>Ov/rf   c            	          t        j                         } t        | d      \  }}}d}t        ||t	        j
                  t        | j                              | j                        }||k(  sJ y )NFrh   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
ry   rz   r   rG   re   r   rK   rL   r|   rz   r   rc   rb   r   r   r   s         rd   %test_classification_report_multiclassr'  S  sl    D'UCFFA
O #yyT../0&&	F _$$$rf   c                  >    g dg d}} d}t        | |      }||k(  sJ y )N)	r   r   r   rF   rF   rF   r=   r=   r=   r  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r   )rc   rb   r   r   s       rd   .test_classification_report_multiclass_balancedr)  m  s/    02MFF
O #662F_$$$rf   c                  x    t        j                         } t        | d      \  }}}d}t        ||      }||k(  sJ y )NFrh   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r   rG   re   r   r&  s         rd   :test_classification_report_multiclass_with_label_detectionr+    sF    D'UCFFA
O #662F_$$$rf   c            	          t        j                         } t        | d      \  }}}d}t        ||t	        j
                  t        | j                              | j                  d      }||k(  sJ y )NFrh   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r   )ry   rz   digitsr%  r&  s         rd   1test_classification_report_multiclass_with_digitsr.    so    D'UCFFA
O #yyT../0&&F _$$$rf   c                      t        d      \  } }}t        j                  g d      |    } t        j                  g d      |   }d}t        | |      }||k(  sJ d}t        | |g d      }||k(  sJ y )NFr   )bluegreenreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r   rz   re   rK   r   r   )rc   rb   r   r   r   s        rd   7test_classification_report_multiclass_with_string_labelr5    s    'u5FFAXX./7FXX./7F
O #662F_$$$
O #66PF_$$$rf   c                      t        d      \  } }}t        j                  g d      }||    } ||   }d}t        | |      }||k(  sJ y )NFr   )u   blue¢u   green¢u   red¢u                precision    recall  f1-score   support

       blue¢       0.83      0.79      0.81        24
      green¢       0.33      0.10      0.15        31
        red¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r4  rc   rb   r   ry   r   r   s         rd   8test_classification_report_multiclass_with_unicode_labelr8    sW    'u5FFAXX:;FF^FF^F
O #662F_$$$rf   c                      t        d      \  } }}t        j                  g d      }||    } ||   }d}t        | |      }||k(  sJ y )NFr   )r0  greengreengreengreengreenr2  a                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r4  r7  s         rd   <test_classification_report_multiclass_with_long_string_labelr;    sW    'u5FFAXX23FF^FF^F
O #662F_$$$rf   c                      g d} g d}g d}d}t        j                  t        |      5  t        | |ddg|       d d d        y # 1 sw Y   y xY w)	Nr   r   r=   r   r   r   r=   r=   r   r   zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3r   r   r=   r$  )r   r  r  r   )rc   rb   rz   r   s       rd   =test_classification_report_labels_target_names_unequal_lengthr@    sB    FF4L
BC	k	-ffaV,W 
.	-	-s   AAc                      g d} g d}g d}d}t        j                  t        |      5  t        | ||       d d d        y # 1 sw Y   y xY w)Nr=  r>  r?  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterr   r3  )r   r   r   r   )rc   rb   rz   r   s       rd   @test_classification_report_no_labels_target_names_unequal_lengthrB    sC    FF4L	. 
 
z	1ff<H 
2	1	1s   AAc                  ~    d} d}t        d|| d      \  }}t        d|| d      \  }}d}t        ||      }||k(  sJ y )Nr   r  rF   r   )r\   r[   r  rB   a                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )r  r[   r   rc   rb   r   r   s          rd   %test_multilabel_classification_reportrD    s_    II.	YQIAv /	YQIAvO #662F_$$$rf   c                     t        j                  g dg dg      } t        j                  g dg dg      }t        | |      dk(  sJ t        | |       dk(  sJ t        ||      dk(  sJ t        |t        j                  |            dk(  sJ t        | t        j                  |             dk(  sJ t        | t        j                  | j
                              dk(  sJ t        |t        j                  | j
                              dk(  sJ y )Nr   r   r   r   r   rF   )rK   r   r%   r   r   rJ   r   s     rd   $test_multilabel_zero_one_loss_subsetrF  5  s    	9i(	)B	9i(	)BR C'''R A%%%R A%%%R^^B/0A555R^^B/0A555RXXbhh/0A555RXXbhh/0A555rf   c                     t        j                  g dg dg      } t        j                  g dg dg      }t        j                  ddg      }t        | |      dk(  sJ t        | |       dk(  sJ t        ||      dk(  sJ t        |d|z
        dk(  sJ t        | d| z
        dk(  sJ t        | t        j                  | j                              dk(  sJ t        |t        j                  | j                              d	k(  sJ t        | ||
      dk(  sJ t        | d|z
  |
      dk(  sJ t        | t        j
                  |       |
      dk(  sJ t        | d   |d         t        | d   |d         k(  sJ y )Nr   r   r   rF   rC   UUUUUU?r   r   r   rG  gUUUUUU?gUUUUUU?)rK   r   r   r   rJ   
zeros_like
sp_hamming)r   r   ws      rd   test_multilabel_hamming_lossrL  C  sm   	9i(	)B	9i(	)B
!QAB5(((B1$$$B1$$$AF#q(((AF#q(((BHHRXX./5888BHHRXX./3666Ba0H<<<AF!4	AAABMM"-Q?7JJJ1r!u%BqE2a5)AAAArf   c                     t        j                  g d      } t        j                  g d      }t        j                  d      }t	        j
                  t        |      5  t        | |dd       d d d        t        j                  g dg dg      } t        j                  g d	g d
g      }d}t	        j
                  t        |      5  t        | |dd       d d d        t        j                  g d      } t        j                  g d      }d}t	        j
                  t        |      5  t        | |d       d d d        d}t	        j
                  t        |      5  t        | |d       d d d        d}t	        j                  t        |      5  t        | |dd       d d d        y # 1 sw Y   AxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   dxY w# 1 sw Y   y xY w)N)r   rF   r   rF   rF   z;pos_label=2 is not a valid label. It should be one of [0 1]r   rX   r=   r   r   r   r   r   r   Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r   )r   rF   rF   r   r=   r  Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r   zJSamplewise metrics are not available outside of multilabel classification.r   zNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r   rC   )
rK   r   reescaper   r   r   r   r  r  )rc   rb   r   msg1msg2msg3r   s          rd   test_jaccard_score_validationrV  W  sn   XXo&FXXo&FiiUVG	z	1ffh!D 
2 XXy),-FXXy),-F	6 	
 
z	.ffh"E 
/ XXo&FXXo&F	 	
 
z	.ffh7 
/WD	z	.ffi8 
/	  
k	-ffgC 
.	-A 
2	1 
/	. 
/	. 
/	. 
.	-s<   F.
F;-G!GG.F8;GGGG(c           	         t        j                  g dg dg      }t        j                  g dg dg      }t        ||d      dk(  sJ t        ||d      dk(  sJ t        ||d      dk(  sJ t        |t        j                  |      d      dk(  sJ t        |t        j                  |      d      dk(  sJ t        |t        j                  |j
                        d      dk(  sJ t        |t        j                  |j
                        d      dk(  sJ t        j                  g dg d	g      }t        j                  g d
g dg      }t        t        ||d      d       t        t        ||d      d       t        t        ||d      d       t        t        ||dddg      d       t        t        ||dddg      d       t        t        ||d       t        j                  g d             t        j                  g dg dg      }t        j                  g d
g dg      }t        t        ||d      d       t        t        ||d      d       d}t        j                  t        |      5  t        ||dgd       d d d        d}t        j                  t        |      5  t        ||dgd       d d d        d}t        j                  t        |      5  t        t        j                  ddgg      t        j                  ddgg      d      dk(  sJ 	 d d d        d}t        j                  t        |      5  t        t        j                  ddgddgg      t        j                  ddgddgg      d      dk(  sJ 	 d d d        t        |       rJ y # 1 sw Y    xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   <xY w)Nr   r   r   r   r   r   rF   r   r   r   r   r   r   r   g?r=   r   r   )r   r   r   r   r   g      ?z	Got 4 > 2r   r   r   z
Got -1 < 0r   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)rK   r   r   r   r   rJ   r4   r6   r   r   r   r  r   rR  )recwarnr   r   rc   rb   rT  rU  r   s           rd   test_multilabel_jaccard_scorerY    sS   	9i(	)B	9i(	)B
 R3t;;;R3q888R3q888R^^B/CqHHHR^^B/CqHHHRXXbhh/CqHHHRXXbhh/CqHHHXXy),-FXXy),-FffgFPffgFPffiH(SffiAG ffiAG ffd3RXX>U5V XXy),-FXXy),-FffgFPffjI7SD	z	.ffaS'B 
/D	z	.ffbT7C 
/	- 
 
,C	8"((QF8,bhhAx.@'R	
 
9	, 
 
,C	81a&1a&)*1a&1a&)*!
 	
 
9 G}}A 
/	. 
/	. 
9	8 
9	8s2    N/6N<,A OAO/N9<OOOc           	         g d}g d}g d}t               }|j                  |       |j                  |      }|j                  |      }t        t        ||      }t        t        ||      }ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]2  }t        |	|
      D ]!  \  }}t         |||       |||             # 4 t        j                  ddgddgddgg      }t        j                  ddgddgddgg      }t               5  t	        ||d      dk(  sJ 	 d d d        t        |       rJ y # 1 sw Y   xY w)N)antr[  catr\  r[  r\  birdr]  )r\  r[  r\  r\  r[  r]  r]  r\  )r[  r]  r\  r[  r]  r\  r   rF   r=   )r   r   r   Nr   r   r   )r*   rT   	transformr   r   zipr4   rK   r   r7   rR  )rX  rc   rb   ry   lbr   r   multi_jaccard_scorebin_jaccard_scoremulti_labels_listbin_labels_listr   m_labelb_labels                 rd   test_multiclass_jaccard_scorerg    su   GFGF#F		BFF6Nf%Jf%J!-@z:F						 1v1v1vsQC!dCO 8 #$5 GGW#GGD!''B !H 8 XX1v1v1v./FXX1v1v1v./F		VVZ@AEEE 
 G}} 
	s   D88Ec                    t        dgdgd      dk(  sJ d}t        j                  t        |      5  t        ddgddgd      dk(  sJ 	 d d d        t        dgdgdd      d	k(  sJ t	        j
                  g d
      }t	        j
                  g d      }t        t        ||d      d       t        t        ||dd      d       t        |       rJ y # 1 sw Y   xY w)NrF   r   rX   r   r   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesr   r  r   )rF   r   rF   rF   r   )rF   r   rF   rF   rF   r   rN  r   )r   r   r  r   rK   r   r4   rR  )rX  r   rc   rb   s       rd   !test_average_binary_jaccard_scoreri    s    !qc84;;;	'  
,C	8aVaVX>#EEE 
9 !qcQASHHHXXo&FXXo&FffhGQffh!Dg G}} 
9	8s   CC c                  (   t        j                  g dg dg      } t        j                  g dg dg      }d}t        j                  t        |      5  t        | |dd      }|t        j                  d      k(  sJ 	 d d d        y # 1 sw Y   y xY w)	Nr   r   r   r   zJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.r   r   r   r   r   r   )rK   r   r   r  r   r   r   )rc   rb   r   r  s       rd   (test_jaccard_score_zero_division_warningrm    sy     XXy),-FXXy),-F	C 
 
,C	8ffivVc**** 
9	8	8s   *BBzzero_division, expected_scorer  )rF   r   c                 H   t        j                  g dg dg      }t        j                  g dg dg      }t        j                         5  t        j                  dt
               t        ||d|       }d d d        t        j                  |      k(  sJ y # 1 sw Y   $xY w)Nr   rk  r   r   rl  )	rK   r   r   r   r   r   r   r   r   )r   expected_scorerc   rb   r  s        rd   *test_jaccard_score_zero_division_set_valuerp    s     XXy),-FXXy),-F		 	 	"g'=>FI]
 
#
 FMM.1111 
#	"s   *BB!c                  h   t        j                  g dg dg dg      } t        j                  g dg dg dg      }t        | |d       \  }}}}t        |g dd       t        |g dd       t        |g d	d       t        |g d
d       t	        | |dd       }|}t        |g dd       t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  |             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      d|z  |z  d|z  |z   z         t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  ||             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      d       y )NrF   r   r   r   r   rF   r   r   r  rF   r   rF   r   r   )r   r   r   r   r=   )r   r   r   r   )r   r   rF   r   )rF   rF   rF   rF   r   r   )r   r  rF   r   r   g      ?r   g?r   r   r   r   r  r   rK   r   r"   r5   r   r4   r   r   rc   rb   r]   r   r   r   f2rm   s           rd   +test_precision_recall_f1_score_multilabel_1ry  $  s$   
 XX|\<@AFXX|\<@AF0NJAq!Q a!5q9a!5q9a!7;aq1	VV!T	:BGb/15 1QJAq!Q7#3+,99FFG<bggbk
 1QJAq!Q33399FFG<	!a1q519% 1TJAq!Q7#3+,99FFJ?


2w' 1SJAq!Q33399FFINPSTrf   c                  j   t        j                  g dg dg dg      } t        j                  g dg dg dg      }t        | |d       \  }}}}t        |g dd       t        |g d	d       t        |g d
d       t        |g dd       t	        | |dd       }|}t        |g dd       t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      d|z  |z  d|z  |z   z         t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  |             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      t        j                  ||             t        | |d      \  }}}}t        |d       t        |d       t        |d       |J t        t	        | |dd      dd       y )Nrr  rs  r  r   r   r   rF   r  r   )r   r   r   r   r=   )r   r   r   r   )r   gQ?r   r   rF   r=   rF   r   ru  )r   皙?r   r   r         ?r   r   r   g      ?rH  r   r   rn   r  r   g&S?rv  rw  s           rd   +test_precision_recall_f1_score_multilabel_2r  g  s'    XX|\<@AFXX|\<@AF 1NJAq!Qa!5q9a!5q9a!6:aq1	VV!T	:BGb/150QJAq!Q4 4 0199FFG<	!a1q519%
 1QJAq!Q4 5!6"99FFG<bggbk 1TJAq!Q5!5!=)99FFJ?


2w'
 1SJAq!Q 5!5!=)99FFI>rf   z%zero_division, zero_division_expected)r   r   r  c           	         t        j                  g dg dg dg      }t        j                  g dg dg dg      }t        ||d |       \  }}}}t        ||dddgd	       t        |dd
d|gd	       d}t        ||dd|gd	       t        |g dd	       t	        ||d	d |       }	|}
t        |	|dd|gd	       t        ||d|       \  }}}}t        j
                  |      rdn|}dt        j
                  |       z   }t        |d	|z   |z         t        |d|z   |z         d}t        ||       |J t        t	        ||d	d|       t        |	d              t        ||d|       \  }}}}t        |d       t        |d
       t        |d       |J t        t	        ||d	d|       d|z  |z  d|z  |z   z         t        ||d|       \  }}}}t        ||dk(  rdnd       t        |d
       d}t        |d|z         |J t        t	        ||d	d|       t        |	|
             t        ||d      \  }}}}t        |d       t        |d       t        |d       |J d }t        t	        ||d	d|       |d	       y )!Nrs  rr  r  ra  r{  rl  r   r   r=   r   r   r   rF   r|  r   r   r   r}  r   rC         ?g?r  r   rq   r   r   r   r   g@r   r   rn   gZd;O?)rK   r   r"   r5   r   r  r4   r8   )r   zero_division_expectedrc   rb   r]   r   r   r   
expected_frx  rm   value_to_sumvalues_to_averageexpected_results                 rd   7test_precision_recall_f1_score_with_an_empty_predictionr    s    XX|\<@AFXX|\<@AF 1MJAq!Q a"8#sC!H!La#sC1G!H!LJa*gq*!EqIaq1	VV!T	WBGb:tQ
"CQG0}JAq!Q !781>TL*@!AABA,0AABC,.2CCD J:&99'	
 	B%	 1}JAq!Q 5!30199FG=	
 
!a1q519%	 1
-JAq!Q $:a$?5SI3M->>?99FJm	
 	B(	 1SJAq!Q 5!5!5!99OFI]	
 		rf   r   )r   r   r   r   c                    t        j                  d      }t        j                  |      }t        j                         5  t        j
                  d       t        |||| |      \  }}}}t        ||| ||      }	d d d        J t        j                  |      r#	fD ]  }
t        j                  |
      rJ  y t        |      }t        |       t        |       t        |       t        	t        |             y # 1 sw Y   xY w)Nrr   rC   r   r   r   r   r  )rK   r   rI  r   r   r   r"   r   r  r   r4   )r   r   r   rc   rb   r]   r   r   r   fbetar   s              rd   "test_precision_recall_f1_no_labelsr    s     XXgF]]6"F		 	 	"g&4'

1a '
 
#" 99 
xx!Q&F88F### '-(M=)=)=)u]34= 
#	"s   ;DDc                    t        j                  d      }t        j                  |      }t        }t	        j
                  t              5   |||| d      \  }}}}d d d        t        d       t        d       t        d       J t	        j
                  t              5  t        ||| d      }d d d        t        d       y # 1 sw Y   oxY w# 1 sw Y   "xY w)Nr  r   r  r   )	rK   r   rI  r"   r   r  r   r4   r   )	r   rc   rb   funcr]   r   r   r   r  s	            rd   1test_precision_recall_f1_no_labels_check_warningsr  3  s    XXgF]]6"F*D	,	-&&'D
1a 
. 11199	,	-FFG#F 
. q! 
.	- 
.	-s   
C
%C
CCc                    t        j                  d      }t        j                  |      }t        j                         5  t        j
                  d       t        ||d d|       \  }}}}t        ||dd |       }d d d        t        j                  |       } t        | | | gd       t        | | | gd       t        | | | gd       t        g dd       t        | | | gd       y # 1 sw Y   nxY w)Nr  r   r   r  r  r=   rk  )
rK   r   rI  r   r   r   r"   r   r  r5   )r   rc   rb   r]   r   r   r   r  s           rd   /test_precision_recall_f1_no_labels_average_noner  G  s    XXgF]]6"F 
	 	 	"g&4'

1a Fd-
 
# JJ}-Ma-!NPQRa-!NPQRa-!NPQRaA.em]M%RTUV) 
#	"s   ;C''C0c                     t        j                  d      } t        j                  |       }t        j                  t
              5  t        | |d d      \  }}}}d d d        t        g dd       t        g dd       t        g dd       t        g dd       t        j                  t
              5  t        | |dd       }d d d        t        g dd       y # 1 sw Y   xY w# 1 sw Y   %xY w)Nr  rF   r  rk  r=   ru  )	rK   r   rI  r   r  r   r"   r5   r   )rc   rb   r]   r   r   r   r  s          rd   4test_precision_recall_f1_no_labels_average_none_warnr  k  s    XXgF]]6"F 
,	-4FDq

1a 
.
 aA.aA.aA.aA.	,	-FFDA 
. eY2 
.	- 
.	-s   C6C*C'*C3c            	         t         t        }} dD ]d  }d}t        j                  ||      5   | g dg d|       d d d        d}t        j                  ||      5   | g dg d|       d d d        f d}t        j                  ||      5   | t	        j
                  d	d
gd	d
gg      t	        j
                  d	d
gd
d
gg      d       d d d        d}t        j                  ||      5   | t	        j
                  d	d
gd
d
gg      t	        j
                  d	d
gd	d
gg      d       d d d        d}t        j                  ||      5   | t	        j
                  d	d	gd	d	gg      t	        j
                  d
d
gd
d
gg      d       d d d        d}t        j                  ||      5   | t	        j
                  d
d
gd
d
gg      t	        j
                  d	d	gd	d	gg      d       d d d        d}t        j                  ||      5   | d	d	gddgd       d d d        d}t        j                  ||      5   | ddgd	d	gd       d d d        t        j                  d      5 }t        j                  d       t        d
d
gd
d
gd       d}t        |j                         j                        |k(  sJ d}t        |j                         j                        |k(  sJ d}t        |j                         j                        |k(  sJ 	 d d d        y # 1 sw Y   xY w# 1 sw Y   "xY w# 1 sw Y   ixY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   sxY w# 1 sw Y   OxY w# 1 sw Y   +xY w# 1 sw Y   y xY w)NNr   r   zPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.r   r   rF   rF   r=   r   zRecall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.rF   r   r   zRecall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r   zRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r   rX   Tr   r   F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r"   r   r   r  rK   r   r   r   r   r   popr   )r   rK  r   r   r   s        rd   test_prf_warningsr    s@   *,BqA. 	 \\!3'iG4 ( 	 \\!3'iG4 ('! /*	  
as	#	"((QFQF#
$bhhAA/?&@)T 
$	  
as	#	"((QFQF#
$bhhAA/?&@)T 
$
	  
as	#	"((QFQF#
$bhhAA/?&@'R 
$	  
as	#	"((QFQF#
$bhhAA/?&@'R 
$
	  
as	#	1a&2r(H- 
$	  
as	#	2r(QFH- 
$ 
	 	 	-h''AAI 	
 6::<''(C/// 	 6::<''(C/// 	 6::<''(C///- 
.	-K (' (' 
$	# 
$	# 
$	# 
$	# 
$	# 
$	# 
.	-sl   LL'>L4.>M>M,>MM(<M5*B&NL$	'L1	4L>MMM%(M25M?Nc           	         t        j                         5  t        j                  d       dD ](  }t        g dg d||        t        g dg d||        * t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d	|        t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d	|        t        ddgd
d
gd|        t        d
d
gddgd|        d d d        t        j                  d      5 }t        j                  d       t        ddgddgd|        t        |      dk(  sJ 	 d d d        y # 1 sw Y   bxY w# 1 sw Y   y xY w)Nr   r  r   r  rl  rF   r   r   r   r   rX   Tr   r   )r   r   r   r"   rK   r   r|   )r   r   r   s      rd   )test_prf_no_warnings_if_zero_division_setr    s   		 	 	"g& 3G+9g] ,9g] 3 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(HHq!fq!f%&HHq!fq!f%&'		
 	(FRHhm	
 	(Hq!fhm	
a 
#h 
	 	 	-h''FQFHM	
 6{a 
.	-i 
#	"h 
.	-s   E-G# 9G/#G,/G8c           	         t        j                         5  t        j                  d       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        d d d        t        j                  d      5 }t        j                  d       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        | d	k(  r(t        |j                         j                        d
k(  sJ t        |      dk(  sJ t        ddgddg       | d	k(  r(t        |j                         j                        d
k(  sJ d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   rF   r   r   rl  Tr   r   r   r  )
r   r   r   r$   rK   r   r   r  r   r|   r   r   s     rd   test_recall_warningsr  -	  sc   		 	 	"g&HHq!fq!f%&HHq!fq!f%&'		
 
# 
	 	 	-h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###aVaV$F"FJJL(() ." ""+ 
.	- 
#	" 
.	-s   AE$
CE0$E-0E9c           	         t        j                  d      5 }t        j                  d       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        | dk(  r(t        |j                         j                        d	k(  sJ t        |      dk(  sJ t        ddgddg       | dk(  r(t        |j                         j                        d	k(  sJ d d d        t        j                         5  t        j                  d
       t        t	        j
                  ddgddgg      t	        j
                  ddgddgg      d|        d d d        y # 1 sw Y   }xY w# 1 sw Y   y xY w)NTr   r   rF   r   r   rl  r   r  r   )
r   r   r   r#   rK   r   r   r  r   r|   r  s     rd   test_precision_warningsr  U	  sc   		 	 	-h'HHq!fq!f%&HHq!fq!f%&'		
 F"FJJL(() ." "" v;!###AA'F"FJJL(() ." ""+ 
.6 
	 	 	"g&HHq!fq!f%&HHq!fq!f%&'		
 
#	"7 
.	-6 
#	"s   CE$AE0$E-0E9c           
         t        j                  d      5 }t        j                  d       t        t	        t
        d      fD ]  } |t        j                  ddgddgg      t        j                  ddgddgg      d| 	       t        |      dk(  sJ  |t        j                  ddgddgg      t        j                  ddgddgg      d| 	       t        |      dk(  sJ  |t        j                  ddgddgg      t        j                  ddgddgg      d| 	       | d
k(  r*t        |j                         j                        dk(  rJ t        |      dk(  rJ  	 d d d        y # 1 sw Y   y xY w)NTr   r   r=   r   rF   r   r   rl  r   r  )r   r   r   r   r   r   rK   r   r|   r   r  r   )r   r   r  s      rd   test_fscore_warningsr  }	  sp   		 	 	-h'! <=E1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 v;!###1a&1a&)*1a&1a&)*+	 &

,,- 2- -- 6{a'''? > 
.	-	-s   D6E.E.!E..E7c                  x   g d} g d}d}t        j                  g dg dg dg      }t        j                  g dg dg dg      }d}| ||f|||ffD ]Y  \  }}}t        t        t        t        t        d	
      fD ]/  }	t        j                  t        |      5   |	||       d d d        1 [ y # 1 sw Y   >xY w)N)rF   r=   rC   rC   )rF   r=   rC   rF   rP  r   r   r   r6  rO  r=   r   r   )
rK   r   r#   r$   r   r   r   r   r   r   )
	y_true_mc	y_pred_mcmsg_mc
y_true_ind
y_pred_indmsg_indrc   rb   r   r   s
             rd   'test_prf_average_binary_data_non_binaryr  	  s    II	1 
 9i;<J9i;<J	<  
Iv&	Z) 
 Ka(	
F z5vv& 65
	  65s   
B00B9c                     d} d}d}d}d}d}| t        j                  ddgddgddgg      f|g d	f|g d
f|g df|t        j                  dgdgdgg      f|t        j                  dgdgdgg      f|t        j                  dgdgdgg      f|t        j                  ddgddgddgg      f|t        j                  ddgddgddgg      fg	}i | | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t        |d      D ]  \  \  }}	\  }
}	 |||
f   }|t	        j
                  t              5  t        |	|       d d d        ||
k7  rCdj                  ||
      }t	        j
                  t        |      5  t        |	|       d d d        |||| fvsdj                  |      }t	        j
                  t        |      5  t        |	|       d d d        t        |	|      \  }}}}}||k(  sJ |j                  d      r"|j                  dk(  sJ |j                  dk(  s@J t        |t        j                  |	             t        |t        j                  |             t	        j
                  t              5  t        |	d d |       d d d         ddg}	ddg}d }t	        j
                  t        |      5  t        |	|       d d d        y # t        $ r ||
|f   }Y w xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y xY w)!Nmultilabel-indicator
multiclassrX   
continuouszmulticlass-multioutputzcontinuous-multioutputr   rF   )r=   rC   rF   r   )r   r  r   r=   rC   r   r  r   r   rZ  g?g      @)r  z@Classification metrics can't handle a mix of {0} and {1} targetsr   z{0} is not supported
multilabelcsrr   )rF   r=   )r   r=   rC   )r=   )r   r=   zYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)rK   r   r   KeyErrorr   r   r   r&   format
startswithr6   squeeze)INDMCBINCNTMMCMCNEXAMPLESEXPECTEDtype1r   type2r   r  r   merged_typer   y1outy2outr   s                      rd   test__check_targetsr  	  sR    !C	B
C
C
"C
"C 
bhhAAA/01	Y	i	o	RXXsQC!o&'	bhhaS1#'(	bhhuse,-.	bhhAAA/01	bhhc
S#Jc
;<=H	c
C	R" 
c
C 
S	4	
 
c
D 
b	2 
c
D 
c
D 
c
D 
c
D 
S	4 
c
D 
c
D 
c
D  
c
D!" 
S	4#$ 
c
D%& 
c
D	c
D	S	4	c
D-H2 %,HQ$?$? [eR	.u-H z*r2& + ~--3VE5-A  ]]:W="2r* >= b#.4;;EBGzA&r2. BA /=R.D+KE5!(***%%l3||u,,,||u,,,"5"**R.9"5"**R.9z*r#2w+ +*A %@H )	B
B	3  
z	-r2 
.	-S  	.u-H	. +* >= BA +* 
.	-sN   L5L,<L9M?M?M L)(L),L6	9M	M	M	 M)c                      d} t        j                  t        t        j                  |             5  t        t        j                  g       t        j                  g              d d d        y # 1 sw Y   y xY w)NzIFound empty input array (e.g., `y_true` or `y_pred`) while a minimum of 1r   )r   r   r   rQ  rR  r&   rK   r   r  s    rd   *test__check_targets_raises_on_empty_inputsr  $
  sC    
UC	z3	8rxx|RXXb\2 
9	8	8s   3A--A6c                  <    ddg} ddg}t        | |      d   dk(  sJ y )Nr   rF   r   r  )r&   rc  s     rd   Atest__check_targets_multiclass_with_both_y_true_and_y_pred_binaryr  *
  s.    VFWF&&)!,<<<rf   zy, target_typerX   r  r   r6  r7  r  c                     |dv r1t        j                  t        d      5  t        | |        ddd       yt        | |       \  }}}}}|dk(  sJ |j                  dk(  sJ |j                  dk(  sJ y# 1 sw Y   yxY w)z?Check correct behaviour when different target types are sparse.)rX   r  z+Sparse input is only supported when targetsr   Nr  r  )r   r   	TypeErrorr&   r  )rZ   target_typey_typer   
y_true_out
y_pred_outs         rd   !test__check_targets_sparse_inputsr  1
  s     ..]]J
 1a 
 
 0>a/C,:z1////  E)))  E)))
 
s   A22A;c                      t        j                  g d      } t        j                  g d      }t        | |      dk(  sJ t        j                  g d      } t        j                  g d      }t        | |      dk(  sJ y )N)r   rF   rF   r   )g      !r   r  g333333ӿr   )r   r=   r=   r   )rK   r   r   rc   pred_decisions     rd   test_hinge_loss_binaryr  I
  sf    XXn%FHH34Mfm,777XXl#FHH34Mfm,777rf   c            
         t        j                  g dg dg dg dg dg dg      } t        j                  g d      }t        j                  d| d   d   z
  | d   d   z   d| d   d   z
  | d   d	   z   d| d	   d	   z
  | d	   d
   z   d| d
   d   z
  | d
   d	   z   d| d   d
   z
  | d   d	   z   d| d   d	   z
  | d   d
   z   g      }t        j                  |dd |       t        j                  |      }t	        ||       |k(  sJ y )N
ףp=
?(\ſ(\gGz)HzGgGz׿Q޿r  333333r  RQؿr  )r  r  r  r  gzGgHzGgHzGѿgQ?)r   rF   r=   rF   rC   r=   rF   r   r=   rC   r   r   outrK   r   clipr   r   )r  rc   dummy_lossesdummy_hinge_losss       rd   test_hinge_loss_multiclassr  S
  sJ   HH((((((	
	M XX()F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,fm,0@@@@rf   c                      t        j                  g d      } t        j                  g dg dg dg dg      }d}t        j                  t        |      5  t        | |       d d d        y # 1 sw Y   y xY w)N)r   rF   r=   r=   )gRQ?g rh?g(\gffffffr  r  zDPlease include all labels in y_true or pass labels as third argumentr   )rK   r   r   r   r   r   )rc   r  error_messages      rd   :test_hinge_loss_multiclass_missing_labels_with_labels_noner  n
  s`    XXl#FHH((((		
M 	O  
z	76=) 
8	7	7   A,,A5c            
         t        j                  g d      } t        j                  g d      }d}t        j                  t        t        j                  |            5  t        | |       d d d        t        j                  ddgddgddgddgddgddgddgg      }g d	}d
}t        j                  t        t        j                  |            5  t        | ||       d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N)r=   rF   r   rF   r   rF   rF   )r   rF   r=   rF   r   r=   rF   zThe shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)r   r  r   rF   r=   r   zThe shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))rc   r  ry   )rK   r   r   r   r   rQ  rR  r   )rc   r  r  ry   s       rd   <test_hinge_loss_multiclass_no_consistent_pred_decision_shaper  
  s     XX+,FHH23M	  
z=)A	B&> 
C HHq!fq!fq!fq!fq!fq!fqRSfUVMF	  
z=)A	B&fM 
C	B 
C	B 
C	Bs   C+C7+C47D c            	         t        j                  g dg dg dg dg dg      } t        j                  g d      }t        j                  g d      }t        j                  d| d   d   z
  | d   d   z   d| d   d   z
  | d   d   z   d| d   d   z
  | d   d	   z   d| d	   d   z
  | d	   d   z   d| d
   d   z
  | d
   d	   z   g      }t        j                  |dd |       t        j                  |      }t	        || |      |k(  sJ y )Nr  皙r  r  r  r  )r   rF   r=   rF   r=   )r   rF   r=   rC   rF   r   r=   rC   r   r  r   r  r  rc   ry   r  r  s        rd   .test_hinge_loss_multiclass_with_missing_labelsr  
  s5   HH(((((	
M XXo&FXXl#F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,fmF;?OOOOrf   c            	         t        j                  g dg dg dg dg dg      } t        j                  g d      }t        j                  g d      }t        j                  d| d   d   z
  | d   d   z   d| d   d	   z
  | d   d   z   d| d	   d	   z
  | d	   d   z   d| d
   d   z
  | d
   d	   z   d| d   d	   z
  | d   d   z   g      }t        j                  |dd |       t        j                  |      }t	        t        || |      |       y )N)r  r  r  )g333333ÿr  r  )r  r  r  )r  g(\gzGڿ)r   r=   r=   r   r=   r   rF   r   r=   rC   r   r  r   )rK   r   r  r   r4   r   r  s        rd   @test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_truer  
  s6   
 HH!!!!!	
M XXo&FXXi F88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
L GGL!T|4ww|,6=8:Jrf   c            
         g d} g dg dg dg dg dg dg}t        j                  d|d   d   z
  |d   d   z   d|d   d   z
  |d   d   z   d|d   d   z
  |d   d	   z   d|d	   d   z
  |d	   d   z   d|d
   d	   z
  |d
   d   z   d|d   d   z
  |d   d	   z   g      }t        j                  |dd |       t        j                  |      }t	        | |      |k(  sJ y )N)r0  r1  r2  r1  whiter2  r  r  r  r  rF   r   r=   rC   r   r   r  r  )rc   r  r  r  s       rd   +test_hinge_loss_multiclass_invariance_listsr  
  s6    ?F$$$$$$M 88a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99a ##mA&6q&99	
	L GGL!T|4ww|,fm,0@@@@rf   c            	      &   g d} t        j                  ddgddgddgddgddgd	d
gg      }t        | |      }t        j                  t	        j
                  t        j                  |       dk(  |d d df                }t        ||       g d} g dg dg dg}t        | |d      }t        |d       | dz  } |dz  }t        | |d      }t        |d       g d} ddgddgddgg}t        j                  t              5  t        | |       d d d        g d} g dg dg dg}g d}d }t        j                  t        t        j                  |      !      5  t        | ||"       d d d        g d#} ddgddgddgddgg}t        | |      }t        |d$       ddg} d%d&gddgg}t        j                  ddgddgg      }d'}t        j                  t        t        j                  |      !      5  t        | |       d d d        d%d&gddgddgg}d(}t        j                  t        t        j                  |      !      5  t        | |       d d d        t        j                  t        j                  |d d df                }t        | |ddg"      }t        ||       g d)} g d*g dg d+g}	t        | |	g d,"      }t        |t        j                  d              y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)-Nnor  r  yesr  r  r   r   rp   {Gz?Gz?r   r~  gMbP?g+?r  rF   rI  r   r   r   r   r   r   r   r   r   TrP  g躕ʀ?r=   Fg.L`@r   r   r   r   r   )rp   r   r   )r   rp   r   r   r   r   )r   r   r   zPy_true contains values {'b'} not belonging to the passed labels ['a', 'c', 'd'].r   r   hamspamr  r  CT?r   r   zy_true contains only one label (2). Please provide the list of all expected class labels explicitly through the labels argument.zBFound input variables with inconsistent numbers of samples: [3, 2]r  )r   r   r   r   r   r   rH  )rK   r   r   r   r
   logpmfr3   r   r   r   rQ  rR  log)
rc   y_probaloss	loss_truery   	error_strr   true_log_losscalculated_log_lossy_score2s
             rd   test_log_lossr  
  s   4Fhh
sc3Z$SzD$<%QVXG FG$D))"((6*:e*CWQPQT]STTID)$ FAGFGt4DD)$ aKFqLGFGu5DD-( FSzC:Sz2G	z	"! 
# FAGF	"  
z9)=	>0 
? ,FSzC:SzC:>GFG$DD)$ VFSzC:&Ghhc
S#J/0G	H  
z9)=	>! 
? SzC:Sz2GTI	z9)=	>! 
?
 WWRVVGAqDM233M"67Aq6B'7 F/BHFHY7DD266#;,'_ 
#	" 
?	>$ 
?	>
 
?	>s0    K!K.0K; L!K+.K8;LLc                     t        j                  ddg|       }t        j                  ddg|       }t        ||      }t        j                  |      sJ y)zCheck the behaviour internal eps that changes depending on the input dtype.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   rF   r  N)rK   r   r   isfinite)rS  rc   r  r	  s       rd   test_log_loss_epsr  =  sJ     XXq!fE*Fhh1vU+GFG$D;;trf   c                     t        j                  g d      }t        j                  ddgddgddgddgg|       }t        j                  t        d	
      5  t        ||       ddd       y# 1 sw Y   yxY w)zHCheck that log_loss raises a warning when y_proba values don't sum to 1.r  r   r   r   r   r   r   r  z$The y_prob values do not sum to one.r   N)rK   r   r   r  r  r   )rS  rc   r  s      rd   'test_log_loss_not_probabilities_warningr  K  sa     XXl#Fhhc
S#Jc
S#JGuUG	k)O	P! 
Q	P	Pr  zy_true, y_probar   r   c                 L    t        | |      t        j                  d      k(  sJ y)z6Check that log_loss returns 0 for perfect predictions.r   N)r   r   r   rc   r  s     rd   !test_log_loss_perfect_predictionsr  U  s"     FG$a(8888rf   c                  H   t        j                  g d      } t        j                  ddgddgddgddgg      }t        t        fg}	 ddlm}m} |j                  ||f       |D ]-  \  }} ||        ||      }}t        ||      }	t        |	d       / y # t        $ r Y >w xY w)	Nr  r   r   r   r   r   )	DataFramer  r  )
rK   r   r1   r  r  r  r  ImportErrorr   r3   )
y_try_prtypesr  r  TrueInputTypePredInputTyperc   r  r	  s
             rd   test_log_loss_pandas_inputr   c  s    8823D88c3Z#sc3Z#sDED]+,E,fi() ).$}'-}T/B(i(	 ).  s   B 	B! B!c                      t        j                  d      } t        j                  t        |       5  t        g dg dg dg dgg d       d d d        y # 1 sw Y   y xY w	NzLabels passed were ['spam', 'eggs', 'ham']. But this function assumes labels are ordered lexicographically. Pass the ordered labels=['eggs', 'ham', 'spam'] and ensure that the columns of y_prob correspond to this ordering.r   eggsr  r  r   r6  r   )r  r$  r  r   )rQ  rR  r   r  r  r   expected_messages    rd   test_log_loss_warningsr'  u  sK    yy	= 
k)9	:#	9-*	
 
;	:	:   AAc                     t        j                  g d      } t        j                  ddgddgddgddgg      }t        | |       d}t        j                  t
        t        j                  |      	      5  t        | |
       ddd       d}t        j                  t        t        j                  |      	      5  t        | ||       ddd       y# 1 sw Y   QxY w# 1 sw Y   yxY w)z?Test `y_pred` deprecation in favor of `y_proba` for `log_loss`.r  r   rp   r   r   ffffff??E`y_pred` was renamed to `y_proba` in version 1.9 and will be removed r   rb   N@Cannot use both `y_pred` and `y_proba`. `y_pred` is deprecated, rb   r  )
rK   r   r   r   r  FutureWarningrQ  rR  r   r   rc   r  r   s      rd    test_log_loss_y_pred_deprecationr2    s    XXl#Fhhc
S#Jc
T4LIJG VW
QC	m299S>	:( 
; MC	z3	89 
9	8	 
;	: 
9	8   5C;CCC(c                     t        j                  g d      } t        j                  g d      }t        j                  | |z
        dz  t	        |       z  }t        t        | |       d       t        t        | |      |       t        t        d| z   |      |       t        t        d| z  dz
  |      |       t        j                  d|z
  |f      }t        j                  d| z
  | f      }t        t        | |      |       t        t        ||      |       t        t        | |d      |       t        t        | |d	      |       t        t        | |d
      d|z         t        t        dgdg      d       t        t        dgdg      d       t        t        dgdg      d       t        t        dgdgd      d       t        t        dgdgd      d       y )Nr   rF   rF   r   rF   rF   r   r   rp   r   r   gffffff?r=   r   r   rF   auto)scale_by_halfTFr   r   g|Gz?r   r  foobarr   )rK   r   r   normr|   r4   r   column_stack)rc   y_prob
true_scorey_prob_reshapedy_true_reshapeds        rd   test_brier_score_loss_binaryrA    s   XX()FXX56FVf_-2S[@J(8#>(8*E(vv>
K(Va@*M ooq6z6&:;Oooq6z6&:;O(A:N(/JJW v>
 t<j u=q:~
 ("u5v>(!se4f=(!se4nE(%3%5I6R%3%59rf   c            	         t        t        g dg dg dg dgg d      d       t        t        g dg dg d	g d
g      d       t        t        g dg dg dg dg      d       t        t        g dg dg dg dg      d       y )Nr#  rr  rs  )r$  r  r  yamsr   r   rI  r  r  r  gt?r   )r   r   r   )r   r   r   )r   r   r   r   r=   )r4   r   r   rf   rd    test_brier_score_loss_multiclassrD    s    #<62	

 	 /J	
 		 /J	
 	
	 /J	
 	
	rf   c                  &   t        j                  g d      } t        j                  g d      }t        j                  t              5  t        | |dd         d d d        t        j                  t              5  t        | |dz          d d d        t        j                  t              5  t        | |dz
         d d d        t        j                  g d      } t        j                  g dg dg dg      }t        j                  t              5  t        | |dd         d d d        t        j                  t              5  t        | |dz          d d d        t        j                  t              5  t        | |dz
         d d d        t        j                  g d	      } t        j                  g d
      }t        j                  d      }t        j                  t        |      5  t        | |       d d d        g d} ddgddgddgg}d}t        j                  t        t        j                  |            5  t        | |       d d d        g d} g dg dg dg}g d}d}t        j                  t        t        j                  |            5  t        | ||       d d d        dg} ddgg}d}t        j                  t        t        j                  |            5  t        | |       d d d        t        t        | |ddg      d       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   uxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   nxY w# 1 sw Y   'xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nr5  r6  rF   r   rI  r  r  r  )r   rF   r=   r   r   r   r   r   zpThe type of the target inferred from y_true is multiclass but should be binary according to the shape of y_prob.r   r   r   zy_true and y_prob contain different number of classes: 3 vs 2. Please provide the true labels explicitly through the labels argument. Classes found in y_true: [0 1 2]r#  r   r6  )r$  r  r  rC  zwThe number of classes in labels is different from that in y_prob. Classes found in labels: ['eggs' 'ham' 'spam' 'yams']r   r$  rp   r   zy_true contains only one label (eggs). Please provide the list of all expected class labels explicitly through the labels argument.r  r  )	rK   r   r   r   r   r   rQ  rR  r4   )rc   r=  r  ry   s       rd   $test_brier_score_loss_invalid_inputsrG    s   XX()FXX56F	z	", 
# 
z	"#. 
# 
z	"#. 
#
 XXi FXXIJF	z	", 
# 
z	"#. 
# 
z	"#. 
#
 XXl#FXX*+FII	AM 
z	7( 
8 F!fq!fq!f%F	  
z=)A	B( 
C %FI.F,F	/ 
 
z=)A	B7 
C XFCj\F	 
 
z=)A	B( 
C (PRVWK 
#	" 
#	" 
#	" 
#	" 
#	" 
#	" 
8	7 
C	B 
C	B 
C	Bsx   L9L *L-L:M2M(M!<M.M;#NL L*-L7:MMM!M+.M8;NNc                      t        j                  d      } t        j                  t        |       5  t        g dg dg dg dgg d       d d d        y # 1 sw Y   y xY wr"  )rQ  rR  r   r  r  r   r%  s    rd   test_brier_score_loss_warningsrI  ,  sQ    yy	= 
k)9	:#
 +	
 
;	:	:r(  c                      d} t        j                  t        |       5  t        g dg d       d d d        y # 1 sw Y   y xY w)Nz%y_pred contains classes not in y_truer   rk  r   )r   r  r  r   r  s    rd   #test_balanced_accuracy_score_unseenrK  ?  s+    
1C	k	-	95 
.	-	-s	   8Azy_true,y_pred)r   r   r   r   )r   r   r   r   )r   r   r   r   c                 T   t        | |dt        j                  |             }t               5  t	        | |      }d d d        t        j                  |      k(  sJ t	        | |d      }t	        | t        j                  | | d               }|||z
  d|z
  z  k(  sJ y # 1 sw Y   exY w)Nr   r   T)adjustedr   rF   )r$   rK   uniquer7   r   r   r   	full_like)rc   rb   macro_recallbalancedrM  chances         rd   test_balanced_accuracy_scorerS  E  s      		&0AL 
	*66: 
 v}}\2222&vvEH$VR\\&&)-LMF6)a&j9999 
	s   BB'r   ))FTr  )r   r   )zeroonec                 8   t         j                  j                  d      }d|d   }}|j                  ||d      }| t        u r|j                  |      }n|j                         } | |||      }t        j                  t        j                  |            rJ y)	zCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    *   ro  r   T)r  replacer  r   N)	rK   rO   rP   choicer   uniformr  anyr  )r   r   r^   r[   r   rc   rb   r  s           rd   *test_classification_metric_pos_label_typesr\  Z  s    * ))


#Cwr{yIZZiZ>F!!),FFi8Fvvbhhv&''''rf   zy_true, y_pred, expected_scorec                 P    t        | |d      t        j                  |      k(  sJ y)zCheck the behaviour of `zero_division` for f1-score.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26965
    r   r  N)r   r   r   )rc   rb   ro  s      rd   2test_f1_for_small_binary_inputs_with_zero_divisionr^  {  s$     FF#6&--:WWWWrf   scoringr  )r   r   c                     t        j                  d      \  }}t        dd      j                  ||      }t	        |||| dd       y)	aZ  Check that we validate `np.nan` properly for classification metrics.

    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
    different in the sub-process and we should not use the `is` operator but
    `math.isnan`.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27563
    r   )rB   rC   )	max_depthrB   r=   r  )r_  n_jobserror_scoreN)r   make_classificationr,   rT   r)   )r_  rY   rZ   
classifiers       rd   :test_classification_metric_division_by_zero_nan_validationrf    sC    ( ''Q7DAq'!!DHHANJJ1gaWUrf   c            	         g d} g d}t        j                  ddgddgddgddgdd	gd
dgg      }t        j                  ddgddgddgddgddgddgg      }t        | |      }t        | |d      }t        | |d      }d||z  z
  }|t	        j
                  |      k(  sJ t        j                  g d      }|d d j                         |j                         z  |d d df<   |dd  j                         |j                         z  |d d df<   t        | ||      }t        | ||d      }t        | ||d      }d||z  z
  }|t	        j
                  |      k(  sJ t        j                  ddgddgddgddgddgddgg      }t        | |      }d|cxk  rdk  sJ  J t        ||      }	|	t	        j
                  |      k(  sJ t        j                  ddgddgddgddgddgddgg      }t        | |      }|dk  sJ t        ||      }	|	t	        j
                  |      k(  sJ g d} t        j                  ddgddgddgddgddgddgg      }t        | |      }|dk(  sJ t        ||      }	|	dk(  sJ g d} g d}t        j                  ddgddgddgddgg      }t        | |      }|dk(  sJ t        ||      }	|	dk(  sJ g d}t        | ||      }
|
dk(  sJ g d} g d}t        j                  g d g d g d!g d"g      }t        | |      }d|cxk  rdk  sJ  J t        | ||      }d|cxk  rdk  sJ  J t        j                  g d#g d$g d"g d%g      }t        | |      }|dk  sJ t        | ||      }|dk  sJ y )&Nrd  r  r   rp   r   r   r   r*  r+  r  r  r  F)rc   r  rQ  rF   )r=   rF   rC   r   rC   rF   rC   r   rc   r  r:  )rc   r  r:  rQ  r   r   r   r   r~  )r   rF   rF   rF   )r  r  r  r  )r=   r=   r=   r=   rG  )highri  lowneutral)ffffff?r   r   r   )r   r   r   r   r   )r   r   r   r  r  )rK   r   r(   r   r   r   r[  )rc   y_true_stringr  y_proba_nulld2_scorelog_likelihoodlog_likelihood_nulld2_score_truer:  d2_score_stringd2_score_with_sample_weights              rd   test_d2_log_loss_scoreru    s   F;Mhh#J#J#J#J4L4L	
	G 88#J#J#J#J#J#J	
	L !@HVWNN"&,RWX)<<<Mv}}]3333 HH/0M&r*..0=3D3D3FFLA&qr*..0=3D3D3FFLA wmH #	N ##	 )<<<Mv}}]3333 hh#J#J#J#J#J#J	
	G !1HC'w?OfmmH5555 hh#J#J#J#J4L#J	
	G !1Ha<<'w?OfmmH5555  Fhh#J#J#J#J#J#J	
	G !1Hq=='w?Oa F/MhhttTlT4L4,OPG 1Hq=='w?Oa M"3}# '!+++ 0F(Mhh		
G !1HC NHChh		
G !1Ha<< NHa<<rf   c                      g d} g d}g d}t        j                  g dd      }t        | |||      }t        j                  g dd      }t        | |||      }d||z  z
  }t        | |||      }t	        ||       y	)
zCheck that d2_log_loss_score works when not all labels are present in y_true

    non-regression test for https://github.com/scikit-learn/scikit-learn/issues/30713
    r=   r   r=   r   r   )rl  r   r   r   r   r   rF   )r:  ry   )r   r   r   rF   N)rK   tiler   r(   r3   )	rc   ry   r:  r  log_loss_obsrn  log_loss_nullexpected_d2_scorero  s	            rd   %test_d2_log_loss_score_missing_labelsr}  >  s    
 FF(Mggi(GFG=QWXL 77=&1LM&M L=88 }VH H/0rf   c                      g d} t        j                  g dd      }t        | |g d      }t        | |g d      }t        ||       y)zGCheck that d2_log_loss_score doesn't depend on the order of the labels.rw  r   rx  r   r   r/  N)rK   ry  r(   r3   )rc   r  ro  d2_score_others       rd   "test_d2_log_loss_score_label_orderr  [  s=    Fggi(G CH&vwyINHn-rf   c                     g d} ddgddgddgg}d}t        j                  t        |      5  t        | |       d	d	d	       g d} ddgddgddgg}g d}d
}t        j                  t        |      5  t        | ||       d	d	d	       g d} g dg dg}d}t        j                  t        |      5  t        | |       d	d	d	       dg} ddgg}d}t        j                  t
        |      5  t        | |       d	d	d	       g d} ddgddgddgg}d}t        j                  t        |      5  t        | |       d	d	d	       g d} dg}ddgddgddgg}d}t        j                  t        |      5  t        | ||       d	d	d	       y	# 1 sw Y   QxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   y	xY w)zPTest that d2_log_loss_score raises the appropriate errors on
    invalid inputs.r   r   r   r   r   r   z#contain different number of classesr   Nz(number of classes in labels is differentr   )r   r   r   )r   r   r   rF  rF   zscore is not well-definedr   y_true contains only one labelz.The labels array needs to contain at least two)r   r   r   r(   r  r   )rc   r  errry   s       rd   test_d2_log_loss_score_raisesr  f  s    FSzC:Sz2G
/C	z	-&'* 
.
 FSzC:Sz2GF
4C	z	-&'&9 
. F0G
+C	z	-&'* 
. SFSzlG
%C	,C	8&'* 
9 FSzC:Sz2G
*C	z	-&'* 
.
 FSFSzC:Sz2G
:C	z	-&'&9 
.	-O 
.	- 
.	- 
.	- 
9	8 
.	- 
.	-sG   F	2F2F#,F/-F;1G	FF #F,/F8;GGc                     t        j                  g d      } t        j                  ddgddgddgddgg      }t        | |       d}t        j                  t
        t        j                  |      	      5  t        | |
       ddd       d}t        j                  t        t        j                  |      	      5  t        | ||       ddd       y# 1 sw Y   QxY w# 1 sw Y   yxY w)zHTest `y_pred` deprecation in favor of `y_proba` for `d2_log_loss_score`.r  r   rp   r   r   r*  r+  r,  r   r-  Nr.  r/  )
rK   r   r(   r   r  r0  rQ  rR  r   r   r1  s      rd   )test_d2_log_loss_score_y_pred_deprecationr    s    XXl#Fhhc
S#Jc
T4LIJG fg&
QC	m299S>	:&1 
; MC	z3	8&'B 
9	8	 
;	: 
9	8r3  c                     g d} g d}g d}g d}g d}t        ||      }t        ||      }t        ||      }d||z  z
  }t        j                  |      |k(  sJ g d}t        ||      }|dk(  sJ t        ||d	
      }|dk(  sJ g d}t        |||       }|dk(  sJ t        ||| d	      }|dk(  sJ g d} g d}g d}g dg dg dg dg dg dg dg dg dg dg
}t        |||       }|dk(  sJ t        |||       }|dk(  sJ g dg dg dg dg dg dg dg dg dg dg
}t        |||       }|dkD  sJ t        |||       }|dkD  sJ y)zeTest that d2_brier_score gives expected outcomes in both the binary and
    multiclass settings.
    )r=   r=   rC   rF   rF   rF   )r   rF   rF   r   r   rF   )r  r  r  r  r  r  )r   r   r   r   rp   r   )r   r   r   r   r   r   r  rF   r   r  )rc   r  r   )r   r   r   r   r   r   rh  )rc   r  r:  r   )
r=   rF   rC   rF   rF   r=   rF   r   rF   r   )
rC   rC   r=   r=   r=   rF   rF   rF   rF   r   )
ddr  ccr  r  bbr  r  r  aa)r   r   r~  r  )r   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   r   N)r'   r   r   r   )	r:  rc   rm  r  y_proba_refro  brier_score_modelbrier_score_refd2_score_expecteds	            rd   test_d2_brier_scorer    s   
 'MF;M -G0KVW=H(H&fkJO-??=="&7777 -GVW=Hq==]GuUHq==
 -GwmH q==#	H q== 3M+FPM 	G wmH q==#H
 q==
 	G wmH c>>#H
 c>>rf   c                     g d} g d}g dg dg dg dg}t        | ||      }|dk(  sJ g d}t        | ||      }|t        j                  |      k(  sJ g dg dg dg dg}t        | ||      }t        j                  |      d	k(  sJ y
)zGTest that d2_brier_score gives expected outcomes when labels are passed)r   r=   r   r=   r   )r   r   r   )rc   r  ry   r   r   r   r   N)r'   r   r   )rc   ry   r  ro  new_d2_scoreneg_d2_scores         rd   test_d2_brier_score_with_labelsr    s    
 FF	G VWVLHq== F!PL6==2222 		G "PL==&",,,rf   z!y_true, y_pred, labels, error_msg)rF   r=   rF   rC   rF  z7inferred from y_true is multiclass but should be binary)r  r  r  r  zpos_label is not specified)r   rF   r   r   rF   rF   r   z.variables with inconsistent numbers of samples)r   rF   r   rF   )g?r   r   r   z%y_prob contains values greater than 1)gr   r   r   z"y_prob contains values less than 0r   r  rt  )r=   rC   rC   r=   )r   r   r   r   )r   r   r   r   z(Multioutput target data is not supportedr   )r   r   r   z"not belonging to the passed labelsrk  z*labels array needs to contain at least twoc                     t        j                  |       } t        j                  |      }t        j                  t        |      5  t        | ||       ddd       y# 1 sw Y   yxY w)zMTest that d2_brier_score raises the appropriate errors
    on invalid inputs.r   r   N)rK   asarrayr   r   r   r'   )rc   rb   ry   	error_msgs       rd   test_d2_brier_score_raisesr  ,  sG    | ZZFZZF	z	3vvf5 
4	3	3s   AA'c                      t        j                  dg      } t        j                  dg      }d}t        j                  t        |      5  t        | |       ddd       y# 1 sw Y   yxY w)zQTest that d2_brier_score emits a warning when there are less than
    two samplesrF   r   z+not well-defined with less than two samplesr   N)rK   r   r   r  r   r'   )rc   rb   warning_messages      rd   4test_d2_brier_score_warning_on_less_than_two_samplesr  p  sM     XXqc]FXXse_FCO	,O	Dvv& 
E	D	Ds   
A  A)z(array_namespace, device_name, dtype_namec                 r   t        | ||      \  }}|j                  g d|      }|j                  g d|      }|j                  g d|      }t        d      5  t        |||      }t	        |      d   t	        |      d   k(  sJ t        |      t        |      k(  sJ 	 ddd       y# 1 sw Y   yxY w)	zTest that `confusion_matrix` works for all array types when `labels` are passed
    such that the inner boolean `need_index_conversion` evaluates to `True`.rH  r-   )r   r   r   T)array_api_dispatchr   r   N)r2   r  r   r   r/   array_api_device)	array_namespacedevice_name
dtype_namexpr.   rc   rb   ry   r  s	            rd   test_confusion_matrix_array_apir  z  s     &o{JOJBZZ	&Z1FZZ	&Z1FZZ	&Z1F	4	0!&&@V$Q'=+@+CCCC'+;F+CCCC 
1	0	0s   AB--B6)NF)rQ  r   	functoolsr   	itertoolsr   r   r   numpyrK   r   scipyr   r   scipy.spatial.distancer	   rJ  scipy.statsr
   sklearnr   r   sklearn.baser   sklearn.calibrationr   sklearn.datasetsr   sklearn.exceptionsr   sklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   sklearn.metrics._classificationr&   r'   r(   sklearn.model_selectionr)   sklearn.preprocessingr*   r+   sklearn.treer,   sklearn.utils._array_apir.   r  r/   r0   sklearn.utils._mockingr1   sklearn.utils._testingr2   r3   r4   r5   r6   r7   sklearn.utils.extmathr8   sklearn.utils.fixesr9   r:   sklearn.utils.validationr;   re   r   markparametrizerr  r   r   r   r   r   r   r   r   r   r   r   r   r  r  r#  r2  rD  rJ  rX  r_  rb  ri  rm  ru  r{  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r"  r'  r)  r+  r.  r5  r8  r;  r@  rB  rD  rF  rL  rV  rY  rg  ri  rm  rp  ry  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  
csr_matrixr  r  r  r  r  r  r  r  r  r  r  float16r  r  r  r   r'  r2  rA  rD  rG  rI  rK  rS  r\  r^  thread_unsaferf  ru  r}  r  r  r  r  r  r  r  r  r   rf   rd   <module>r     s   	   2 2     8 ! ! ' 6 ; 5     . 
 4 @ / 1  . > 7+(d>DB 61a*@A B& T{aVUOi=OP))(7D PQ R$ PQ*J R*JZ PQM RM,>6  BHH####	
	
 .=	
(9)(9  BHH#####	
 	
*;+*;
 =*	= OF .9.9%S : :%SPT< +%%;&) 
 #"((#56""((#56 N	
 #"((#56""((#56
B		
 #"((#56""((#56 N	
 #"((#56""((#56 H	
;$'P*Q'P*  #"((?3""((?3
<		
**"2;0 		S!E"C C 





 $S	!3'	$bff-S	!3'	'0		D	D$ $S	!3'	$c*	$c*	'0		D	D$> 01
 qA37	QC"Hq!fd3
rA38T4(
qA37	QC!GqcAg-1vt< qA37	QC!GqcAg-1vx@" /#rvv?A @# 2&A(1.1 1a.9)aS1#J<8!$	' 9 :' )aS1#J<8!$	 9$2N RF5p c5\2U 3UB7(t $ST, U,0"(".& 	:;
QAB 
'(  886 "AB0 C0"%4%$%(%6 %F%,%.XI PQ% R%<6B(%DPJZ"J.+ 868:LM	2 N	2 PQ?U R?UD PQ= R=@ PQ+&&266266"23[	 R
[| !%$MN1a.9"5 : O &"5J $MN" O"& 1a.9 W : WF38e0P 1a.9:  :: z 61a*@A$ B$N 61a*@A$
 B$
N 61a*@A#( B#(L'>^B3= 			QC!qcA3/	0(;			QC!qcA3/	0,?			Iy)<	=|L** 8A6*"N8P4@A8I(X 2::rzz2::"FG
 H
 2::rzz2::"FG" H" 	I	aVaVaV,-	Y	95699)$
 :"$N FIXX
&6 	34	34	34
:
: #&' C(($ $	1a&	8288QF+S1	1a&	8288QF+S1	1a&	8288QF+S1	1a&	8288QF+S1	XX HBFF3Karvv>O266:L7	V VSl1:..:dC"`F-@ '  E		
 ' (		
 " <		
  3		
 !0		
 3Z#sc3Z0,		
 <(!#786		
 o?F0		
 o?C8		
c7:v6w:v6' .-/D	Drf   