3j7NUddlmZgdZddlZddlZddlZddlZddlZddlm Z ddlm Z e jdZ ddZ Gdd ZGd d ZGd d ej ZGddej$ZgZded<edk(rddlZej.eyy)) annotations) JobProcessorMetadataCachingJob cacheMetadata WorkerProcessN)common) environmentzmetadata.cachingcddlm}|jd}| |ddddgz}tj|s|f}tj }|j g}|D])}|j|}||j|||z }+d|d } |durtj| ntj| |D]6} d | } |durtj| "tj| 8y) zv Cache metadata from corpora in `corpusNames` as local cache files. Call as ``metadata.cacheMetadata()`` r)managerT)skipNoneNlocalcorezcache: final writing time: z secondszpath failed to parse: ) music21.corpusr listLocalCorporaNamesr isIterableTimerstartfromNamer environLocalwarn printDebug) corpusNamesuseMultiprocessingverboser localCorporaNamestimerfailingFilePaths corpusName corpusObjectmessagefailingFilePaths E/DATA/.local/lib/python3.12/site-packages/music21/metadata/caching.pyrr s '55t5D'*gv-@@   [ )"n LLNE KKM" '' 3 L667I7TYZZ",E7(;G$'"(+*?*;< d?   g &  # #G , ,cReZdZdZd dZdZdZdZdZdZ d Z d Z e d Z y) ra  Parses one corpus path, and attempts to extract metadata from it: >>> job = metadata.caching.MetadataCachingJob( ... 'bach/bwv66.6', ... parseUsingCorpus=True, ... corpusName='core', ... ) >>> job.jobNumber 0 >>> job.corpusName 'core' >>> job.run() ((,), ()) >>> results = job.getResults() >>> errors = job.getErrors() TODO: error list, not just numbers needs to be reported back up. Nctj||_g|_t ||_g|_t||_||_ yN) pathlibPathfilePathfilePathErrorsint jobNumberresultsboolparseUsingCorpusr)selfr)r,r/rs r"__init__zMetadataCachingJob.__init__esA X.  Y $%5 6$r#cJddl}g|_|j}tj d|j d||1d|j vr|j|n|j|~|j|j|jfS)NrzGot ParsedObject from z: Opus) gcr- parseFilePathrrr)classes parseOpus parseNonOpuscollect getResults getErrors)r0r4 parsedObjects r"runzMetadataCachingJob.runms ))+ $T]]O2l^ D F  #---|,!!,/   $.."222r#cddlm}ddlm}d} |jdur|j |j d}|S|j t |j d} |S#t$r|}tjd|j d|tjtj|jj|j Yd}~|Sd}~wwxYw) Nr) converter)corpusFT) forceSourcezparse failed: z, )music21r?r@r/parser)str Exceptionrr traceback format_excr*append)r0r?r@r<es r"r5z MetadataCachingJob.parseFilePath|s%"  6$$-(t}}$O  &||C ,>D|Q   6  # #nT]]O2aS$I J  # #I$8$8$: ;    & &t}} 5 5  6s+A&&A&& C+/A1C&&C+cddlm} |jjj |j }|j|j }|j|j|j|tjd||jj|j ||j}|jj|ytjdtj t#|j |jj|j d|j}|jj|y#t$$rMtj'd|j(dtj't+j,YywxYw)NrmetadatazupdateMetadataCache: storing: ) sourcePathmetadataPayloadrz:addFromPaths: got stream without metadata, creating stub: z+Had a problem with extracting metadata for z, piece ignored)rBrLbundlesMetadataBundlecorpusPathToKey cleanFilePath RichMetadatamergeupdaterr MetadataEntryrr-rHr relativepathrDrErr)rFrG)r0r<rL corpusPath richMetadata metadataEntrys r"r8zMetadataCachingJob.parseNonOpuss$ 6!))88HH""$J$$0'446 ""<#8#89##L1''4ZLAC ( 0 0 > >#11$0#!?!  ##M2''&&,&9&9#d>P>P:Q&R%SU!) 0 0 > >#11$(#!?!  ##M2 6   }}o_6    i224 5  6sCE1%B E11AGGc ddlm}d} t|jD]\}}|j ||~ |jj|jd}|jj!|y#t $rY}t jd|d|jd|t jtjYd}~d}~wwxYw)NrrK1Had a problem with extracting metadata for score  in , whole opus ignored: )rMrN)rBrL enumeratescoresparseScoreInsideOpusrErrr)rrFrGrOrVrRr-rH)r0r<rL scoreNumberscore exceptionrZs r"r7zMetadataCachingJob.parseOpuss$  <&/ 0C0C&D" U))%='E!((66)) 7  M* <   C-tDMM?2H+   # #I$8$8$: ; ;  > C ACC c Xddlm} |j}|j|j|j ||j|jj ,t jdt|jzy|jjj|j|jj }t jd||jj|j|jj |}|jj!|y#t"$rY}t j%d|d|jd |t jt'j(Yd}~yd}~wwxYw) NrrKzLaddFromPaths: got Opus that contains Streams that do not have work numbers: )numberzaddFromPaths: storing: )rMrfrNr\r]r^)rBrLrSrTrUrfrrrDr)rOrPrQrRrVr-rHrErrFrG)r0rcrbrLrYrXrZrds r"raz'MetadataCachingJob.parseScoreInsideOpussq % <#002L   u~~ .    &~~%)>)>)F''@$--()&--<<LL&& >>00M ''-j\:< ( 0 0 > >#11 >>00$0!?!  ##M2 <   CK=Qmm_$:9+G   # #I$8$8$: ; ;  >> jobs = [] >>> mdb = corpus.corpora.CoreCorpus().search('monteverdi')[:3] >>> paths = [x.sourcePath for x in mdb] >>> for corpusPath in paths: ... job = metadata.caching.MetadataCachingJob( ... corpusPath, ... parseUsingCorpus=True, ... corpusName='core', ... ) ... jobs.append(job) >>> jobGenerator = metadata.caching.JobProcessor.process_serial(jobs) >>> for result in jobGenerator: ... print(result['remainingJobs']) ... 2 1 0 c,||z }d|d|d|d|}|S)z3 Report on the current job status. zupdated z of z files; total errors: z ... last file: rt) totalJobs remainingJobsr)filePathErrorCount completedJobsr s r"_reportzJobProcessor._report'sA "M1 }oT)5/01" %  r#Nc#K|xstj}t|d}t|}t ||}t j d|d|dg}tj}tj}t|Dcgc]}t||}}|D]}|j|r|D]&} |jtj| (tt|D]]} tj |j#} | j%}| j'} |dz}|| | j(|d_|D]} |jd|j+|j-|j-|D]}|j+ycc}ww)a Process jobs in parallel, with `processCount` processes. If `processCount` is none, use 1 fewer process than the number of available cores. jobs is a list of :class:`~music21.metadata.MetadataCachingJob` objects. z Processing z jobs in parallel, with z processes.metadataEntrieserrorsr)rxN)rcpusmaxlenminrrmultiprocessing JoinableQueueQueuerangerrputpickledumpsloadsgetr:r;r)joinclose) jobs processCountrxr- job_queue result_queue_workersworkerjobunused_jobCounterr_workers r"process_parallelzJobProcessor.process_parallel6s$4v{{} <+ D <7 -(@k Z \#113 &,,. !,/1/q!L9/ 1F LLN  fll3/0%*3t9%5!ll<#3#3#56..*" '.$ # %2  &6G MM$ F KKM/1sBGF>$DGc#Kt|}|D]-}|j\}}|dz}|||j|d/yw)z( Process jobs serially. r}r~N)rr=r))rrxrr-rs r"process_serialzJobProcessor.process_serialgsN D C!ggiOGV Q M#* LL!.  s?Ar&)rzlist[MetadataCachingJob])rorprqrr staticmethodr{rrrtr#r"rrsEB  --`  r#rc(eZdZdZfdZdZxZS)rz[ A worker process for use by the multithreaded metadata-caching job processor. c>t|||_||_yr&)superr1rr)r0rr __class__s r"r1zWorkerProcess.__init__s "(r#cR |jj}||jjytj|}|j |jj|j jtj|d)Nr)protocol) rr task_donerrr=rrr)r0rs r"r=zWorkerProcess.runsz..$$&C{((*,,s#C GGI NN $ $ &    ! !&,,sQ"? @r#)rorprqrrr1r= __classcell__)rs@r"rr|s) Ar#rc eZdZy)TestN)rorprqrtr#r"rrsr#rz list[type] _DOC_ORDER__main__)NTF) __future__r__all__rr'rrFunittestrBrr EnvironmentrrrrProcessrTestCaserr__annotations__romainTestrtr#r"rs#  &{&&'9: #%))-^pplppnAO++A> 8    J zGTr#