
    yjd<                        d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZ d dlmZmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z&m'Z'm(Z(m)Z)m*Z* d dl+m,Z,  e j-        e.          Z/d e j-        d          _0        de	j1        ddfdZ2ddd dde3e         dedz  de3e4         dz  de4dz  de3e5         de6de6d e5ddfd!Z7d"e&d#e4ddfd$Z8d"e(ddfd%Z9dS )&    N)ValidationError)select)crudmodels)
tracked_db)"process_representation_tasks_batch)process_dream)ResourceNotFoundExceptionValidationException)Message)cleanup_queue_items)run_vector_reconciliation_cycle)ReconcilerTypeResolvedConfiguration)CleanupStaleItemsCompletedEventDeletionCompletedEventSyncVectorsCompletedEventemit)log_performance_metrics)
summarizer)DeletionPayloadDreamPayloadReconcilerPayloadSummaryPayloadWebhookPayload)webhook_deliveryTzsqlalchemy.engine.Engine
queue_itemreturnc           	      x
  K   | j         }| j        }| j        }|dk    rt          j        dd          5  	 t          di |}n[# t          $ rN}t                              dt          |          |           t          dt          |                     |d}~ww xY wt          |           d{V  ddd           n# 1 swxY w Y   dS |t          | d          |d	k    r	 t          di |}n[# t          $ rN}t                              d
t          |          |           t          dt          |                     |d}~ww xY wt          j        ||           d{V  dS |dk    rK	 t          di |}n[# t          $ rN}t                              dt          |          |           t          dt          |                     |d}~ww xY w| j        }|t          d          |j        }|sLt                              d|           t'          d          4 d{V }t)          t*          j                                      t*          j        j        |k                                  t*          j        j        |j        k                                  t*          j        j        |k              }	|                    |	           d{V }
|
                                }|/t                              d|           	 ddd          d{V  dS |j        }ddd          d{V  n# 1 d{V swxY w Y   t          j        dd          5  t;          j        ||j        ||j        ||j                    d{V  tC          d| d|            ddd           dS # 1 swxY w Y   dS |dk    rt          j        dd          5  	 tE          di |}n[# t          $ rN}t                              dt          |          |           t          dt          |                     |d}~ww xY wtG          ||           d{V  ddd           dS # 1 swxY w Y   dS |dk    rt          j        dd          5  	 tI          di |}n[# t          $ rN}t                              dt          |          |           t          dt          |                     |d}~ww xY wtK          ||           d{V  ddd           dS # 1 swxY w Y   dS t          d|           )z%Process a single item from the queue.
reconcilerprocess_reconciler_taskderiver)nameopz4Invalid reconciler payload received: %s. Payload: %szInvalid payload structure: Nz tasks require a workspace_namewebhookz1Invalid webhook payload received: %s. Payload: %ssummaryz1Invalid summary payload received: %s. Payload: %sz"Summary tasks require a message_idz)Fetching message public ID for message %ssummary_fallback)operation_namez;Failed to fetch message with ID %s for process_summary_taskprocess_summary_task_dreamprocess_dream_taskz/Invalid dream payload received: %s. Payload: %sdeletionprocess_deletion_taskz2Invalid deletion payload received: %s. Payload: %szInvalid task type:  )&	task_typepayloadworkspace_name
sentry_sdkstart_transactionr   r   loggererrorstr
ValueErrorprocess_reconcilerr   r   deliver_webhookr   
message_idmessage_public_iddebugr   r   r   r   wheresession_nameidexecutescalar_one_or_none	public_idr   summarize_if_neededmessage_seq_in_sessionconfigurationr   r   r	   r   process_deletion)r   r0   queue_payloadr2   	validateder;   r<   dbstmtresultmessages               </DATA/AppData/hermes/projects/honcho/src/deriver/consumer.pyprocess_itemrP   &   s0     $I&M.N L  )/HYWWW 
	0 
	0P->>>>		" P P PJFF!  
 !!Gs1vv!G!GHHaOP %Y/////////
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 
	0 	 IFFFGGGI	L&7777II 	L 	L 	LLLCA  
 C3q66CCDD!K	L .y.IIIIIIIIIII	i			L&7777II 	L 	L 	LLLCA  
 C3q66CCDD!K	L  *
ABBB%7  	6LLDjQQQ!1CDDD 6 6 6 6 6 6 66>**U6>8NJKKU6>6):PPQQU6>,
:;;	   "zz$//////// 3355?LLU"   6 6 6 6 6 6 6 6 6 6 6 6 6 6  %,$5!!6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6$ )/E)TTT 		Q 		Q0&0!'         $I./O/O:/O/OPPP		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 		Q 
g		)/C	RRR 
	; 
	;P(99=99		" P P PEFF!  
 !!Gs1vv!G!GHHaOP  	>:::::::::
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
	; 
j	 	 )/F9UUU 
	> 
	>P+<<m<<		" P P PHFF!  
 !!Gs1vv!G!GHHaOP #9n=========
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> 
	> :y::;;;s  B?AB?
BA	BBB??CC'C4 4
E>A	EE4F 
GA	GG/CL9 L99
MM AN11N58N5Q'O)(Q')
Q3A	P<<QQ''Q+.Q+TRT
S7)A	S22S77TT!$T!F)hit_batch_token_capwas_flush_enabledbatch_max_tokensmessagesmessage_level_configuration	observersobservedqueue_item_message_idsrQ   rR   rS   c          
         K   | r| d         st                               d           dS ||t          |          dk    rt          d          t	          | |||||||           d{V  dS )a[  
    Prepares and processes a batch of messages for representation tasks.

    Args:
        messages: List of messages to process
        message_level_configuration: Resolved configuration for this batch
        observers: List of observers for the messages
        observed: The observed of the messages
        queue_item_message_ids: Message IDs from queue items
        hit_batch_token_cap: whether the queue batcher clamped this batch to fit
        was_flush_enabled: snapshot of DERIVER.FLUSH_ENABLED at fetch time
        batch_max_tokens: DERIVER.REPRESENTATION_BATCH_MAX_TOKENS snapshot
    r   z1process_representation_batch received no messagesNz<observed and observers are required for representation tasks)rV   rW   rX   rQ   rR   rS   )r5   r=   lenr8   r   )rT   rU   rV   rW   rX   rQ   rR   rS   s           rO   process_representation_batchr[      s      0  8A; HIII9,I!0C0CWXXX
,#5/+)	 	 	 	 	 	 	 	 	 	 	 	    r1   r2   c                   K   | j         }| j        }d}d}d}d}d}d}	t                              d|||           	 t	          d          4 d{V }
|dk    r	 t          j        |
||           d{V }|j        }|j        }	t                              d||||	           ny# t          $ r4}t          
                    d	|t          |                     Y d}~n@d}~ww xY w|d
k    r}	 t          j        |
||           d{V  d}	t                              d||           n# t          $ r3}t          
                    d|t          |                     Y d}~nd}~ww xY w|dk    r	 t          j        |
|           d{V }|j        }|j        }|j        }|j        }	t                              d|||||	           nV# t          $ r3}t          
                    d|t          |                     Y d}~nd}~ww xY wd}d| }t!          |          	 ddd          d{V  n# 1 d{V swxY w Y   n0# t"          $ r#}d}|t%          |          j         d| } d}~ww xY wt)          t+          ||||||||	|	  	                   dS # t)          t+          ||||||||	|	  	                   w xY w)a  
    Process a deletion task from the queue.

    This function handles the actual deletion of resources based on the deletion type.
    It is designed to be idempotent - deleting an already-deleted resource is a no-op.

    Args:
        payload: The deletion payload containing deletion_type and resource_id
        workspace_name: The workspace name for scoping the deletion

    Raises:
        ValueError: If the deletion type is not supported
    TNr   z?Processing deletion task: type=%s, resource_id=%s, workspace=%srG   session)r2   r?   zMSuccessfully deleted session %s in workspace %s (messages=%d, conclusions=%d)zASession %s not found during deletion (may already be deleted): %sobservation)r2   document_id   z3Successfully deleted observation %s in workspace %szEObservation %s not found during deletion (may already be deleted): %s	workspace)r2   zVSuccessfully deleted workspace %s (peers=%d, sessions=%d, messages=%d, conclusions=%d)zCWorkspace %s not found during deletion (may already be deleted): %sFzUnsupported deletion type: z: )	r2   deletion_typeresource_idsuccesspeers_deletedsessions_deletedmessages_deletedconclusions_deletederror_message)rc   rd   r5   infor   r   delete_sessionrh   ri   r
   warningr7   delete_document_by_iddelete_workspacerf   rg   r   	Exceptiontype__name__r   r   )r1   r2   rc   rd   re   rj   rf   rg   rh   ri   rK   rM   rJ   s                rO   rG   rG      s     " )M%KG $MM
KKI	  `
011 H	9 H	9 H	9 H	9 H	9 H	9 H	9R	))#'#6>$ $ $      F (.'>$*0*D'KK:#&(+    1   NN[#A        -//4>{          +,'KKM#&   
 1   NN_#A        +--#'#8>$ $ $      F %+$8M'-'>$'-'>$*0*D'KKQ&%((+    1   NN]&A          Mm M M)-8888QH	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9 H	9R    
  #Aww/66166M 	"-+'+!1!1$7+
 
 
	
 	
 	
 	
 	
"-+'+!1!1$7+
 
 
	
 	
 	
 	
s   I H/A	B#!H/#
C!-)CH/C!!	H/+;D'&H/'
E$1)EH/E$$	H/.AGH/
H)G>9H/>HH/I /
H99I <H9=I  J 
I.I))I..J 'J?c           
      :  K   | j         }t          j                    }|t          j        k    rt
                              d           t                       d{V }t          j                    |z
  dz  }|j        dk    s|j	        dk    s|j
        dk    rut
                              d|j        |j        |j        |j        |j                   t#          t%          |j        |j        |j        |j        |j        |                     dS dS |t          j        k    rot
                              d           t)                       d{V }t          j                    |z
  dz  }|dk    r t#          t+          ||                     dS dS t-          d	|           )
aQ  
    Process a reconciler task from the queue.

    Currently supports:
    - sync_vectors: Syncs pending documents/message embeddings to vector store
      and cleans up soft-deleted documents.
    - cleanup_queue: Removes old processed queue items.

    Args:
        payload: The reconciler payload containing the reconciler type
    zProcessing sync_vectors taskNi  r   zvReconciliation complete: synced %s docs, %s message embeddings; failed %s docs, %s message embeddings; cleaned %s docs)documents_synceddocuments_faileddocuments_cleanedmessage_embeddings_syncedmessage_embeddings_failedtotal_duration_mszProcessing cleanup_queue task)queue_items_cleanedry   zUnsupported reconciler type: )reconciler_typetimeperf_counterr   SYNC_VECTORSr5   r=   r   total_syncedtotal_failedtotal_cleanedrk   rt   rw   ru   rx   rv   r   r   CLEANUP_QUEUEr   r   r8   )r1   r{   
start_timemetricsduration_msdeleted_counts         rO   r9   r9   M  s      -O"$$J.5553444799999999(**Z74?  1$$#a''$q((KK I(1(1)   )%,%=%,%=&-&?.5.O.5.O&1  	 	 	 	 	 )(. 
N8	8	84555133333333(**Z74?1/(5&1        JJJKKKr\   ):loggingr|   r3   pydanticr   
sqlalchemyr   srcr   r   src.dependenciesr   src.deriver.deriverr   src.dreamerr	   src.exceptionsr
   r   
src.modelsr   src.reconciler.queue_cleanupr   src.reconciler.sync_vectorsr   src.schemasr   r   src.telemetry.eventsr   r   r   r   src.telemetry.loggingr   	src.utilsr   src.utils.queue_payloadr   r   r   r   r   src.webhooksr   	getLoggerrr   r5   disabled	QueueItemrP   listr7   intboolr[   rG   r9   r/   r\   rO   <module>r      s         $ $ $ $ $ $               ' ' ' ' ' ' B B B B B B % % % % % % I I I I I I I I       < < < < < < G G G G G G = = = = = = = =            : 9 9 9 9 9                          * ) ) ) ) )		8	$	$9=  , - - 6q<6#3 q< q< q< q< q<v !&#( ( (7m(!6!=( Cy4	(
 Dj( !I( ( ( ( 
( ( ( (VE
E
E
 
E
 E
 E
 E
P>L&7 >LD >L >L >L >L >L >Lr\   