
    yj2Q                    z   d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z'  ej(        e)          Z*d3dZ+d4dZ, G d d          Z-dddddddej.        j/        dej0        d
d5d2Z1dS )6    )annotationsN)suppress)Any)select)AsyncSession)crud
exceptionsmodelsschemas)settings)
tracked_db)check_and_schedule_dream)embedding_client)ResolvedConfiguration)EmbeddingCallPurpose)accumulate_metric)format_datetime_utc)DeductiveObservationExplicitObservationRepresentation)embedding_call_purposeobs*ExplicitObservation | DeductiveObservationreturnstrc                H    t          | t                    r| j        n| j        S )zKReturn the canonical text payload for an explicit or deductive observation.)
isinstancer   
conclusioncontent)r   s    ?/DATA/AppData/hermes/projects/honcho/src/crud/representation.py_observation_textr!      s     '-ABBS3>>S    c                    t          |                                           }t          | t                    r|                     d|i          S |                     d|i          S )z?Return an observation with its persisted/embed text normalized.r   )updater   )r!   stripr   r   
model_copy)r   texts     r    _normalized_observationr(   $   sb     S!!''))D#+,, ;~~lD%9~:::>>)T!2>333r"   c                      e Zd ZdZd>dZd?dZd@dZdddddddej        j	        de
j        d
dAd-Zddddddej        j	        d.dBd/Z	 	 	 dCdDd5Z	 dEdFd6ZdGd7ZdHd9Z	 	 dIdJd;Z	 dEdKd=ZdS )LRepresentationManagerz8Unified manager for representation and document queries.workspace_namer   observerobservedr   Nonec               0    || _         || _        || _        d S Nr+   r,   r-   )selfr+   r,   r-   s       r    __init__zRepresentationManager.__init__1   s     $2%%r"   representationr   message_ids	list[int]session_namemessage_created_atdatetime.datetimemessage_level_configurationr   intc           
       K   d}|j         s#|j        st                              d           |S d |j         |j        z   D             }|st                              d           |S t	          j                    }d |D             }		 t          t          j        j	        | j
        d          5  t          j        |	           d{V }
ddd           n# 1 swxY w Y   n<# t          $ r/}t          j        d	t           j        j         d
z             |d}~ww xY wt	          j                    |z
  dz  }t'          d|d          d| j         d|d           t	          j                    }t+          d          4 d{V }|                     |||
||||           d{V }ddd          d{V  n# 1 d{V swxY w Y   t	          j                    |z
  dz  }t'          d|d          d| j         d|d           |S )a  
        Save Representation objects to the collection as a set of documents.

        Args:
            representation: Representation object
            message_ids: Message ID range to link with observations
            session_name: Session name to link with existing summary context
            message_created_at: Timestamp when the message was created

        Returns:
            The number of *new documents saved*
        r   zNo observations to savec                n    g | ]2}t          |                                          #t          |          3S  )r!   r%   r(   .0r   s     r    
<listcomp>z=RepresentationManager.save_representation.<locals>.<listcomp>W   sJ     
 
 
 %%++--
#C((
 
 
r"   z!No non-empty observations to savec                ,    g | ]}t          |          S r>   )r!   r?   s     r    rA   z=RepresentationManager.save_representation.<locals>.<listcomp>c   s!    PPP.s33PPPr"   r4   r+   parent_categoryNz3Observation content exceeds maximum token limit of .i  deriver__embed_new_observationsmsz*representation_manager.save_representationsave_new_observations)	deductiveexplicitloggerdebugtimeperf_counterr   r   CREATE_OBSERVATIONSvaluer+   r   simple_batch_embed
ValueErrorr	   ValidationExceptionr   	EMBEDDINGMAX_INPUT_TOKENSr   r,   r   _save_representation_internal)r2   r4   r5   r7   r8   r:   new_documentsall_observationsbatch_embed_startobservation_texts
embeddingsebatch_embed_durationcreate_document_startdbcreate_document_durations                   r    save_representationz)RepresentationManager.save_representation<   sw     * ' 	!0G 	!LL2333  
 
%/.2II
 
 

   	!LL<===   !-//PP?OPPP	'$8>#2 0    
 $4#F%$ $      
                	 	 	0E'8;;;<  	 !% 1 3 36G G4O8{28888$ 		
 	
 	
 !% 1 3 3JKK 		 		 		 		 		 		 		r"&"D"D "+# #      M		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 %)$5$7$7:O$OSW#W 8{28888#$		
 	
 	
 sN   &C ,CC CC CC 
D)*DD?"F33
F= F=rb   r   r[   0list[ExplicitObservation | DeductiveObservation]r^   list[list[float]]c           
       K   t          j        || j        | j        | j                   d {V }g }	t          ||d          D ]\  }
}t          |
t                    rd}|
j        }|
j	        }nd}|
j
        }d }t          j        ||t          |                    }|	                    t          j        |||||                     t          j        ||	| j        | j        | j        t"          j        j                   d {V }|j        j        rL	 t-          ||           d {V  n4# t.          $ r'}t0                              d	|            Y d }~nd }~ww xY wt5          |          S )
N)r,   r-   T)strictrL   rM   )r5   premisesr8   )r   r7   levelmetadata	embedding)r,   r-   deduplicatez"Failed to check dream scheduling: )r   get_or_create_collectionr+   r,   r-   zipr   r   r   ri   r   r   DocumentMetadatar   appendDocumentCreatecreate_documentsr   DERIVERDEDUPLICATEdreamenabledr   	ExceptionrN   warninglen)r2   rb   r[   r^   r5   r7   r8   r:   
collectiondocuments_to_creater   rl   	obs_levelobs_contentobs_premisesrk   accepted_documentsr_   s                     r    rY   z3RepresentationManager._save_representation_internal   s       8]]	
 
 
 
 
 
 
 
 

 =?!"2JtLLL 	 	NC#344 $'	!n"|&	!k#181I'%#67I#J#J2 2 2H  &&&'!-#%'      $(#8]] (4$
 $
 $
 
 
 
 
 
 
 ',4 	II.r:>>>>>>>>>> I I IGAGGHHHHHHHHI %&&&s   D- -
E7EENF
rb   r7   include_semantic_queryrl   semantic_search_top_ksemantic_search_max_distanceinclude_most_derivedmax_observationsrD   embedding_purposeAsyncSession | None
str | Noner   rl   list[float] | Noner   
int | Noner   float | Noner   boolr   rD   r   r   c       
        
  K   |r||zt          t                    5  t          |
j        | j        |	          5  t          j        |           d{V }ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   |#|                     ||||||||           d{V S t          d          4 d{V 	 }|                     ||||||||           d{V cddd          d{V  S # 1 d{V swxY w Y   dS )am  
        Get working representation with flexible query options.

        Args:
            db: Optional database session. If provided, uses it directly;
                otherwise creates a new session via tracked_db.
            session_name: Optional session to filter by
            include_semantic_query: Query for semantic search
            embedding: Pre-computed embedding for the semantic query.
            semantic_search_top_k: Number of semantic results
            semantic_search_max_distance: Maximum distance for semantic search
            include_most_derived: Include most derived observations
            max_observations: Maximum total observations to return
            parent_category: Optional workflow attribution forwarded to the
                fallback embedding call when the caller didn't pre-compute
                an embedding (or pre-compute failed).
            embedding_purpose: Embedding call_purpose tag to use on the
                fallback embed when no pre-computed embedding was supplied.
                Defaults to SEARCH_MEMORY; callers whose route-level
                precompute uses a more specific purpose (e.g.
                SESSION_CONTEXT_SEARCH) should pass that here so the
                fallback path lands in the same analytics bucket.

        Returns:
            Representation combining various query strategies
        NrC   r7   r   rl   r   r   r   r   z1representation_manager.get_working_representation)	r   rx   r   rS   r+   r   embed$_get_working_representation_internalr   )r2   rb   r7   r   rl   r   r   r   r   rD   r   new_dbs               r    get_working_representationz0RepresentationManager.get_working_representation   s     P " 	Qi&7 ##Q Q&%+#'#6$3  Q Q #3"89O"P"PPPPPPP	Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q Q >BB)'=#&;-I%9!1 C 	 	 	 	 	 	 	 	 	 ?
 
 	 	 	 	 	 	 	 	BB)'=#&;-I%9!1 C 	 	 	 	 	 	 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	sF   A6AA6A#	#A6&A#	'A66A:=A:<#C22
C<?C<r   c                 K   |}	|r%t          t          d||n|	dz            |	          nd}
|r'|r%t          t          d|	dz            |	|
z
            }n)|r%t          t          d|	dz            |	|
z
            }nd}|	|
z
  |z
  }t                      }|rG|                     |||
||           d{V }|                    t          j        |                     |rD|                     ||           d{V }|                    t          j        |                     |                     |||           d{V }|                    t          j        |                     |S )z6Internal implementation of get_working_representation.r   N      )querytop_kmax_distancerl   )r   )r   r7   )minmaxr   _query_documents_semanticmerge_representationfrom_documents_query_documents_most_derived_query_documents_recent)r2   rb   r7   r   rl   r   r   r   r   totalsemantic_observationstop_observationsrecent_observationsr4   semantic_docsderived_docsrecent_docss                    r    r   z:RepresentationManager._get_working_representation_internal&  s?      ! &
C,8 *)!	      	 " 	!&: 	!"3q%1*#5#5u?T7TUU! 	!"3q%1*#5#5u?T7TUU  ! $&;;>NN')) " 
	"&"@"@,+9# #A # #      M //-m<<  
   	!%!C!C* "D " "      L //-l;;  
 !88) 9 
 
 
 
 
 
 
 
 	++N,I+,V,VWWWr"   r   r   r   rj   list[models.Document]c           
     b  K   	 |r!|                      ||||||           d{V S t          j        || j        | j        | j        ||||           d{V }|                                 t          |          S # t          $ r)}t          
                    d|            g cY d}~S d}~ww xY w)z'Query documents by semantic similarity.)rl   N)r+   r,   r-   r   r   r   rl   z%Error getting relevant observations: )_query_documents_for_levelr   query_documentsr+   r,   r-   expunge_alllistrx   rN   error)	r2   rb   r   r   r   rj   rl   	documentsr_   s	            r    r   z/RepresentationManager._query_documents_semanticr  s     	 '!<< ' =          #'"6#'#6!]!]!-'	# 	# 	# 	 	 	 	 	 		    I& 	 	 	LLDDDEEEIIIIII	s#   "A; AA; ;
B.B)#B.)B.c                  K    t          t          j                                      |          j        t          j        j        | j        k    t          j        j        | j        k    t          j        j        | j        k    t          j        j        	                    d          g|t          j        j
        |k    gng R                      t          j        j                                                  }|                    |           d{V }|                                                                }|                                 t%          |          S )zQuery most recent documents.N)r   r
   Documentlimitwherer+   r,   r-   
deleted_atis_r7   order_by
created_atdescexecutescalarsallr   r   )r2   rb   r   r7   stmtresultr   s          r    r   z-RepresentationManager._query_documents_recent  s     
F6?##U5\\.$2EE(DM9(DM9*..t44 $/ _1\ABB   Xfo0557788 	" zz$''''''''NN$$((**	
Ir"   c                  K   t          t          j                                      |                              t          j        j        | j        k    t          j        j        | j        k    t          j        j        | j        k    t          j        j        	                    d                    
                    t          j        j                                                  }|                    |           d{V }|                                                                }|                                 t#          |          S )zQuery most derived documents.N)r   r
   r   r   r   r+   r,   r-   r   r   r   times_derivedr   r   r   r   r   r   )r2   rb   r   r   r   r   s         r    r   z3RepresentationManager._query_documents_most_derived  s      
 6?##U5\\U.$2EE(DM9(DM9*..t44	  Xfo388::;; 	 zz$''''''''NN$$((**	
Ir"   floatc                D   K   |                      |||||           d{V S )z;Internal method that does the actual observation retrieval.N)r   )r2   rb   r   r   r   rj   s         r    _get_observations_internalz0RepresentationManager._get_observations_internal  sI       33ulE
 
 
 
 
 
 
 
 	
r"   countc                   K   t          j        || j        | j        | j        ||||                     |          |	  	         d{V }t          t          |          d d          }|S )z%Query documents for a specific level.)r+   r,   r-   r   r   r   filtersrl   Nc                    | j         S r0   )r   )xs    r    <lambda>zBRepresentationManager._query_documents_for_level.<locals>.<lambda>  s    1< r"   T)keyreverse)r   r   r+   r,   r-   _build_filter_conditionssortedr   )	r2   rb   r   rj   r   r   rl   r   docs_sorteds	            r    r   z0RepresentationManager._query_documents_for_level  s       ..]]%11%88

 

 

 

 

 

 

 

 

	 .4OO!7!7.
 .
 .
 r"   dict[str, Any]c                    i }|r||d<   |S )z
        Build filter conditions for document queries.

        Returns a flat dict of key-value pairs for vector store filtering.
        rj   r>   )r2   rj   r   s      r    r   z.RepresentationManager._build_filter_conditions  s      #% 	%$GGr"   )r+   r   r,   r   r-   r   r   r.   )r4   r   r5   r6   r7   r   r8   r9   r:   r   r   r;   )rb   r   r[   re   r^   rf   r5   r6   r7   r   r8   r9   r:   r   r   r;   )rb   r   r7   r   r   r   rl   r   r   r   r   r   r   r   r   r;   rD   r   r   r   r   r   )rb   r   r7   r   r   r   rl   r   r   r   r   r   r   r   r   r;   r   r   )NNN)rb   r   r   r   r   r;   r   r   rj   r   rl   r   r   r   r0   )rb   r   r   r;   r7   r   r   r   )rb   r   r   r;   r   r   )rb   r   r   r   r   r;   r   r   rj   r   r   r   )NN)rb   r   r   r   rj   r   r   r;   r   r   rl   r   r   r   )rj   r   r   r   )__name__
__module____qualname____doc__r3   rd   rY   r   rt   'WORKING_REPRESENTATION_MAX_OBSERVATIONSr   SEARCH_MEMORYr   r   r   r   r   r   r   r   r>   r"   r    r*   r*   .   s       BB	& 	& 	& 	&T T T Tl?' ?' ?' ?'H #'#'-1(,,059%* ( 0 X&*2F2TO O O O O On $(-1(,,059%* ( 0 XJ J J J J Jb &* (,$ $ $ $ $N HL    4   *
 
 
 
& &*(,    < !      r"   r*   Fr   r+   rb   r   r,   r-   r7   r   r   rl   r   r   r   r   r   r   r   r   r;   rD   r   r   r   c               t   K   t          | ||          }|                    |||||||	|
||
  
         d{V S )a  
    Get raw working representation data from the relevant document collection.

    This is a convenience function that creates a RepresentationManager and calls
    get_working_representation on it.

    Args:
        db: Optional database session. If provided, uses it directly;
            otherwise creates a new session via tracked_db.
        embedding: Pre-computed embedding for the semantic query.
        parent_category: Workflow attribution forwarded to the fallback
            embedding call when no pre-computed embedding was supplied.
        embedding_purpose: Embedding call_purpose for the fallback embed;
            callers should match it to whatever purpose their route-level
            precompute used so failure/retry paths stay in the same bucket.
    r1   r   N)r*   r   )r+   rb   r,   r-   r7   r   rl   r   r   r   r   rD   r   managers                 r    r   r     s      @ $%  G
 33!53%A1)'+ 4         r"   )r   r   r   r   )r   r   r   r   )r+   r   rb   r   r,   r   r-   r   r7   r   r   r   rl   r   r   r   r   r   r   r   r   r;   rD   r   r   r   r   r   )2
__future__r   datetimeloggingrP   
contextlibr   typingr   
sqlalchemyr   sqlalchemy.ext.asyncior   srcr   r	   r
   r   
src.configr   src.dependenciesr   src.dreamer.dream_schedulerr   src.embedding_clientr   src.schemasr   src.telemetry.eventsr   src.telemetry.loggingr   src.utils.formattingr   src.utils.representationr   r   r   src.utils.typesr   	getLoggerr   rN   r!   r(   r*   rt   r   r   r   r>   r"   r    <module>r      s6   " " " " " "                      / / / / / / 1 1 1 1 1 1 1 1 1 1 1 1       ' ' ' ' ' ' @ @ @ @ @ @ 1 1 1 1 1 1 - - - - - - 5 5 5 5 5 5 3 3 3 3 3 3 4 4 4 4 4 4         
 3 2 2 2 2 2		8	$	$T T T T
4 4 4 4P P P P P P P Pr #  $)-$((,15!&$,T"&.B.P0 0 0 0 0 0 0 0r"   