
    S3jO                     @
   d Z ddlZddlZddlmZ ddlZddlmZ 	 ddlZ	 ddl
Z
ddlmZ ddlmZmZmZmZmZmZmZmZmZmZ ddlmZ d	d
lmZmZmZmZmZm Z m!Z!m"Z"  e       Z# e       Z$ e        Z% e!       Z& e"       Z'ejP                  jS                  edu d      ejP                  jU                  de#dz   e%g      d               Z+ejP                  jS                  e
du d      d        Z,d Z-ejP                  j\                  d        Z/ejP                  j\                  d        Z0ejP                  j\                  ejP                  jU                  d ejb                  edejP                  jd                        edfedfgg d      d               Z3ejP                  j\                  ejP                  jU                  d ejb                  e$ejP                  jd                        e%e'gg d      d               Z4ejP                  j\                  d        Z5ejP                  j\                  ejP                  jd                  d               Z6ejP                  j\                  ejP                  jd                  ejP                  jU                  d d!d"g      d#                      Z7ejP                  j\                  d$        Z8ejP                  j\                  ejP                  jS                  e
du d%      d&               Z9ejP                  j\                  ejP                  jS                  e
du d%      d'               Z:ejP                  jS                  e
dud(      d)        Z;ejP                  jS                  edud*      ejP                  jU                  d+eeeg      d,               Z<ejP                  j\                  ejP                  jS                  edu d      ejP                  jU                  d-e#ef ejb                  e$eejP                  jd                        gd.d/g      d0                      Z=ejP                  j\                  ejP                  jS                  edu d      d1               Z>ejP                  j\                  ejP                  jS                  edu d      ejP                  jS                  e
du d      d2                      Z?ejP                  j\                  d3        Z@ G d4 d5      ZA G d6 d7      ZBy# e	$ r dZY w xY w# e	$ r dZ
Y w xY w)8zK
Test the downloader classes and functions separately from the Pooch core.
    N)TemporaryDirectory)	HTTPError   )Pooch)
HTTPDownloaderFTPDownloaderSFTPDownloaderDOIDownloaderchoose_downloaderFigshareRepositoryZenodoRepositoryDataverseRepository
doi_to_urlREQUESTS_HEADERS)Unzip   )pooch_test_urlcheck_large_datacheck_tiny_datadata_over_ftppooch_test_figshare_urlpooch_test_zenodo_url pooch_test_zenodo_with_slash_urlpooch_test_dataverse_urlzrequires tqdm)reasonurltiny-data.txtc                 >    t        | d      }|j                  du sJ y)DThe progressbar keyword argument must pass through choose_downloaderTprogressbarNr   r!   r   
downloaders     I/DATA/.local/lib/python3.12/site-packages/pooch/tests/test_downloaders.pytest_progressbar_kwarg_passedr&   ?   s$     #3D9J!!T)))    zrequires paramikoc                  B    d} t        | d      }|j                  du sJ y)r   /sftp://test.rebex.net/pub/example/pocketftp.pngTr    Nr"   r#   s     r%   "test_progressbar_kwarg_passed_sftpr*   M   s)     <C"3D9J!!T)))r'   c                      t        j                  t              5  t        d       ddd       t        j                  t              5  t        d       ddd       y# 1 sw Y   7xY w# 1 sw Y   yxY w)z6Should raise ValueError when protocol is not supportedzhttpup://some-invalid-url.comNzdoii:XXX/XXX/file)pytestraises
ValueErrorr    r'   r%   test_unsupported_protocolr0   U   sM    	z	"9: 
# 
z	"-. 
#	" 
#	" 
#	"s   AA(A%(A1c                      t        j                  t              5 }  t               ddd       ddd       dt	         j
                        v sJ y# 1 sw Y   #xY w)z/Should fail if data repository is not supportedz%doi:10.21105/joss.01943/file_name.txtN)r   output_filepoochz(Invalid data repository 'joss.theoj.org')r,   r-   r.   r
   strvalueexcs    r%   test_invalid_doi_repositoryr8   ^   sL     
z	"c7TQU	
 
#
 6SYYGGG 
#	"s   AAc                  x    t        j                  t              5  t        d       ddd       y# 1 sw Y   yxY w)z#Should fail if the DOI is not foundNOTAREALDOI)doiN)r,   r-   r   r   r/   r'   r%   test_doi_url_not_foundr<   i   s$     
y	!}% 
"	!	!s   09zrepository,doiz10.6084/m9.figshare.14763051.v1)marksz10.5281/zenodo.4924875z10.11588/data/TKCFEF)figsharezenodo	dataverse)idsc                     t        j                  t              5 }t        |      }| j	                  ||      }|j                  d       ddd       dt        j                        v sJ y# 1 sw Y   #xY w)z3Should fail if the file is not found in the archivezbla.txt	file_nameNzFile 'bla.txt' not found)r,   r-   r.   r   
initializedownload_urlr4   r5   )
repositoryr;   r7   r   repos        r%    test_figshare_url_file_not_foundrI   p   sd      
z	"co$$S#.I. 
# &SYY777	 
#	"s   0A,,A5c                     t               5 }t               }t        j                  j	                  |d      } || dz   |d       t        |       ddd       y# 1 sw Y   yxY w)zTest the DOI downloaderr   N)r   r
   ospathjoinr   )r   local_storer$   outfiles       r%   test_doi_downloaderrP      sL     
	"_
'',,{O<3('48 	 
		s   AAA c                  0   t               5 } t        dz   }t               }t        j                  j                  | d      } |||d        t               |dd      }|D cg c]	  }d|v s| c}\  }t        |       ddd       yc c}w # 1 sw Y   yxY w)zf
    Test the Zenodo downloader when the path contains a forward slash

    Related to issue #336
    z!santisoler/pooch-test-data-v1.zipztest-data.zipNdownload)actionr3   r   )r   ZENODOURL_W_SLASHr
   rK   rL   rM   r   r   )rN   base_urlr$   rO   fnamesffnames          r%   *test_zenodo_downloader_with_slash_in_fnamerY      s     
	$'JJ"_
'',,{O<8Wd+ 4@%>v!A)=Av> 
	 ? 
	s$   AB 	B*B.BBBc                  `   t         } | d| j                  d       dz   } | dd }d| d}t               5 }t               }t        j
                  j                  |d      }t        j                  t        |	      5   || dz   |d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)
zf
    Test if passing a Figshare url without a version warns about it, but still
    downloads it.
    N./   zThe Figshare DOI 'z#' doesn't specify which version of r   match)
FIGSHAREURLrindexr   r
   rK   rL   rM   r,   warnsUserWarning)r   r;   warning_msgrN   r$   rO   s         r%   !test_figshare_unspecified_versionrf      s     C


3
 3
&C
a)C&se+NOK		"_
'',,{O<\\+[9s_,gt< : 
	 :9 
	s$   AB$9BB$B!	B$$B-zversion, missing, present)r   7LC08_L2SP_218074_20190114_20200829_02_T1-cropped.tar.gzcropped-before.tar.gz)r   rh   rg   c                     d|  }d| d}t        ||      }|j                  D cg c]  }|d   	 }}||v sJ ||vsJ yc c}w )zG
    Test if setting the version in Figshare DOI works as expected
    z10.6084/m9.figshare.21665630.vhttps://doi.org/r\   nameN)r   api_response)versionmissingpresentr;   r   r>   item	filenamess           r%   &test_figshare_data_repository_versionsrr      sl    . +7)
4CSE
#C!#s+H*2*?*?@*?$f*?I@i)### As   Ac                    t        | d      5 }t               5 }t        | j                        }t        j
                  j                  |d      } |||d       t        |       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zTest ftp downloaderr   )portN)r   r   r   server_portrK   rL   rM   r   )	ftpserverr   rN   r$   rO   s        r%   test_ftp_downloaderrw      sm     
y/	2c![&I,A,ABJggll;@GsGT*G$	 " 
3	2!! 
3	2s#   BAA5$B5A>	:BB
zrequires paramiko to run SFTPc                      t               5 } t        dd      }d}t        j                  j	                  | d      } |||d       t        j                  j                  |      sJ 	 ddd       y# 1 sw Y   yxY w)zTest sftp downloaderdemopasswordusernamerz   r)   pocketftp.pngN)r   r	   rK   rL   rM   exists)rN   r$   r   rO   s       r%   test_sftp_downloaderr      s_     
	#VjI
?'',,{O<3&ww~~g&&& 
		s   AA00A9c                  R   t               5 } t        dd      }d}t        j                  j	                  | d      }t        |d      5 }t        j                  t              5   |||d       ddd       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w)zFDownloader should fail when a file object rather than string is passedry   rz   r{   r)   r}   wbN)	r   r	   rK   rL   rM   openr,   r-   	TypeError)rN   r$   r   rO   outfile_objs        r%   (test_sftp_downloader_fail_if_file_objectr      s     
	#VjI
?'',,{O<'4 Ky)3T2 * !	 
	
 *) ! 	 
	s;   <BB!B,B4BB
BB	BB&zparamiko must be missingc                      t        j                  t              5 } t                ddd       dt	         j
                        v sJ y# 1 sw Y   #xY w)z+test must fail if paramiko is not installedNz
'paramiko')r,   r-   r.   r	   r4   r5   r6   s    r%   -test_sftp_downloader_fail_if_paramiko_missingr     s<     
z	"c 
#3syy>))) 
#	"s   AAztqdm must be missingr$   c                     t        j                  t              5 } | d       ddd       dt        j                        v sJ y# 1 sw Y   #xY w)zFMake sure an error is raised if trying to use progressbar without tqdmTr    Nz'tqdm')r,   r-   r.   r4   r5   )r$   r7   s     r%   !test_downloader_progressbar_failsr   
  s>     
z	"ct$ 
#s399~%%% 
#	"s   
AAzurl,downloaderhttpr>   c                     |d      }t               5 }d}| |z   } t        j                  j                  ||      } || |d       |j	                         }|j
                  j                  d      d   j                         }t        |      dk(  sJ t        j                  dk(  rd	}	nd
}	|dd |	k(  sJ t        |       ddd       y# 1 sw Y   yxY w)z@Setup a downloader function that prints a progress bar for fetchTr    r   Nr^   O   win32100%|####################A   100%|████████████████████   )r   rK   rL   rM   
readouterrerrsplitstriplensysplatformr   )
r   r$   capsysrR   rN   rX   rO   capturedprintedprogresss
             r%   test_downloader_progressbarr     s     d+H		Ek'',,{E2gt$$$&,,$$T*2.4467|r!!!<<7"2HZHs|x''' ! 
		s   B+CCc                    t        |d      5 }t        d|j                        }t               5 }t        j
                  j                  |d      } |||d       | j                         }|j                  j                  d      d   j                         }t        |      dk(  sJ t        j                  dk(  rd	}nd
}|dd |k(  sJ t        |       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zESetup an FTP downloader function that prints a progress bar for fetchr   T)r!   rt   Nr   r^   r   r   r   r   r   )r   r   ru   r   rK   rL   rM   r   r   r   r   r   r   r   r   )	r   rv   r   rR   rN   rO   r   r   r   s	            r%   test_downloader_progressbar_ftpr   3  s     
y/	2c T	8M8MN![ggll;@GS'4( ((*Hll((.r288:Gw<2%%%||w&6^ 3B<8+++G$! " 
3	2!! 
3	2s#   "C0B$C$C0$C-	)C00C9c                    t        ddd      }t               5 }d}t        j                  j	                  |d      } |||d       | j                         }|j                  j                  d      d	   j                         }t        |      d
k(  sJ t        j                  dk(  rd}nd}|dd |k(  sJ t        j                  j                  |      sJ 	 ddd       y# 1 sw Y   yxY w)zFSetup an SFTP downloader function that prints a progress bar for fetchTry   rz   )r!   r|   rz   r)   r}   Nr   r^   r   r   r   r   r   )r	   r   rK   rL   rM   r   r   r   r   r   r   r   r~   )r   r$   rN   r   rO   r   r   r   s           r%    test_downloader_progressbar_sftpr   L  s    
  D6JWJ		?'',,{O<3&$$&,,$$T*2.4467|r!!!<<7"2HZHs|x'''ww~~g&&& 
		s   B<CC(c                     G d d      } |d      }t        |      }t               5 }d}t        |z   }t        j                  j                  |d      } |||d       | j                         }|j                  j                  d      d   j                         }	d	}
|	|
k(  sJ t        |       ddd       y# 1 sw Y   yxY w)
zASetup a downloader function with an arbitrary progress bar class.c                   >    e Zd ZdZd Zd Zd Zd Zd Ze	d        Z
y)	Etest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplayz&A minimalist replacement for tqdm.tqdmc                      d| _         || _        y )Nr   )counttotal)selfr   s     r%   __init__zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__init__k  s    DJDJr'   c                 ^    t        | j                        dz   t        | j                        z   S )zrepresent current completionr\   )r4   r   r   r   s    r%   __repr__zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__repr__o  s"    tzz?S(3tzz?::r'   c                 B    t        d|  t        j                  d       y)zprint self.__repr__ to stderrr    )fileendNprintr   stderrr   s    r%   renderzLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.renders  s    Btf+CJJB7r'   c                 2    || _         | j                          y)zmodify completion and renderN)r   r   )r   is     r%   updatezLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.updatew  s    DJKKMr'   c                     d| _         y)zset counter to 0r   N)r   r   s    r%   resetzKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.reset|  s	    DJr'   c                  :    t        dt        j                         y)zprint a new empty liner   )r   Nr   r/   r'   r%   closezKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.close  s     "3::&r'   N)__name__
__module____qualname____doc__r   r   r   r   r   staticmethodr   r/   r'   r%   MinimalProgressDisplayr   h  s2    4		;	8	
	 
	' 
	'r'   r   N)r   r    zlarge-data.txtr   r^   z336/336)r   r   BASEURLrK   rL   rM   r   r   r   r   r   )r   r   pbarrR   rN   rX   r   rO   r   r   r   s              r%   %test_downloader_arbitrary_progressbarr   d  s    ' ': "-D$/H		 o'',,{,<=gt$$$&,,$$T*2.446(""" 	! 
		s   BB99Cc                      e Zd ZdZdZde Zde ZdZdZdZ	ddeeed	ed
e	 deidgdZ
ddeeed	ee	deidgdZddeeed	ee	deidd	ed
e	 deidgdZej                  j                  dde
fdefdefg      d        Zej                  j                  dde
fdefg      d        Zej                  j                  de
eg      d        Zy)TestZenodoAPISupportz0
    Test support for different Zenodo APIs
    i@ z10.0001/zenodo.rj   zmy-file.zipzMhttps://zenodo.org/api/files/513d7033-93a2-4eeb-821c-2fb0bbab0012/my-file.zip 2942bfabb3d05332b66eb128e0842cffz 2021-20-19T08:00:00.000000+00:00z$513d7033-93a2-4eeb-821c-2fb0bbab0012md5:r   )idkeychecksumlinks)createdmodifiedr   r;   doi_urlfiles)r   filenamer   r   zapi_version, api_responselegacynewinvalidc                 v   |j                  d| j                         j                  |       t        | j                  | j
                        }|j                  d      |_        |dk7  r|j                  |k(  sJ yd}t        j                  t        |      5  |j                  }ddd       y# 1 sw Y   yxY w)z@
        Test if the API version is correctly detected.
        /zenodo.r;   archive_urlr   r   z0Couldn't determine the version of the Zenodo APIr_   N)expect_request
article_idrespond_with_jsonr   r;   r   url_forbase_api_urlapi_versionr,   r-   r.   )r   
httpserverr   rl   r$   msgs         r%   test_api_versionz%TestZenodoAPISupport.test_api_version  s     	!!HT__,=">?QQ	
 &$((M
","4"4R"8
)#))[888DCz5(44 655s   B//B8c                 z   |j                  d| j                         j                  |       t        | j                  | j
                        }|j                  d      |_        |j                  | j                        }|dk(  r|| j                  k(  sJ y	d| j                   d| j                   d}||k(  sJ y	)
zK
        Test if the download url is correct for each API version.
        r   r   r   rC   r   zhttps://zenodo.org/records/z/files/z?download=1N)r   r   r   r   r;   r   r   r   rF   rD   file_url)r   r   r   rl   r$   rF   expected_urls          r%   test_download_urlz&TestZenodoAPISupport.test_download_url  s     	!!HT__,=">?QQ	
 &$((M
","4"4R"8
!...H("4==000 .??#74>>*:+G   <///r'   rl   c                 \   |j                  d| j                         j                  |       t        d|      }t	        | j
                  | j                        }|j                  d      |_        |j                  |       |j                  | j                  d| j                   ik(  sJ y)zX
        Test if population of registry is correctly done for each API version.
        r   r   )rU   rL   r   r   N)r   r   r   r   r   r;   r   r   r   populate_registryregistryrD   file_checksum)r   r   tmp_pathrl   puppyr$   s         r%   test_populate_registryz+TestZenodoAPISupport.test_populate_registry  s     	!!HT__,=">?QQ	
 r1%$((M
","4"4R"8
$$U+~~$..D9K9K8L2M!NNNNr'   N)r   r   r   r   r   r;   r   rD   r   r   legacy_api_responsenew_api_responseinvalid_api_responser,   markparametrizer   r   r   r/   r'   r%   r   r     s    JJ<
(C &GIW  7M 66 = "=/2H		
& 66 =%)H		
& 66 =%)H	 = "=/2H	
4 [[#*+$%,-	
55( [[#
'	(52B*CD0	0. [[	./O	Or'   r   c                   "    e Zd ZdZd Zd Zd Zy)TestDOIDownloaderHeadersz+Test the headers argument in DOIDownloader.c                     t               }|j                  t        k(  sJ t        d      }|j                  t        k(  sJ y)z#Test the default value for headers.Nheaders)r
   r   r   r   r$   s     r%   test_default_headersz-TestDOIDownloaderHeaders.test_default_headers4  s=    "_
!!%5555"40
!!%5555r'   c                 J    t        ddi      }ddi}|j                  |k(  sJ y)zTest overwriting for headers.customfieldr   Nr
   r   )r   r$   expected_headerss      r%   test_overwrite_headersz/TestDOIDownloaderHeaders.test_overwrite_headers;  s5    "Hg+>?
g
 !!%5555r'   c                 >    t        i       }|j                  i k(  sJ y)z&Test passing an emtpy dict to headers.r   Nr   r   s     r%   test_headers_empty_dictz0TestDOIDownloaderHeaders.test_headers_empty_dictC  s     "2.
!!R'''r'   N)r   r   r   r   r   r   r   r/   r'   r%   r   r   1  s    566(r'   r   )Cr   rK   r   tempfiler   r,   requestsr   tqdmImportErrorparamikor   r   downloadersr   r   r	   r
   r   r   r   r   r   r   
processorsr   utilsr   r   r   r   r   r   r   r   r   ra   	ZENODOURLrT   DATAVERSEURLr   skipifr   r&   r*   r0   networkr8   r<   paramr>   rI   rP   rY   rf   rr   rw   r   r   r   r   r   r   r   r   r   r   r/   r'   r%   <module>r     s|   
 
 '  
    	 	 	 
%'!#	46 ') DDL9	/!* :* H$-@A* B*/ H H & & -++&&	

 
34	45 	,  8 8 	V\\+V[[%9%9:I|T+  
! !  & =  =$ 	

	
$  "$ % % H$-LM' N ' H$-LM3 N 3 HD(1KL* M* D$-CD~'VW& X E& DDL9	.![-v{{7K7KL 
  ! : !, DDL9% : %. DDL9H$-@A' B : '* /" /"dWO WOt( (w  D
  Hs"   T T TTTT