
    yjp.                        d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ d dlmZmZ d dlmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z(m)Z) d dl*m+Z+ ddl,m-Z-m.Z.  e j/        e0          Z1defdZ2 e"dd          ddd dde3e         dedz  de3e4         d e4d!e3e5         d"e6d#e6d$e5ddfd%            Z7dS )&    N)crud)ConfiguredModelSettingssettings)RepresentationManager)
tracked_db)honcho_llm_call)LLMTelemetryContext)Message)ResolvedConfiguration)prometheus_metrics)RepresentationCompletedEventemit)CallPurpose)accumulate_metriclog_performance_metrics)DeriverComponentsDeriverTaskTypes
TokenTypes)with_sentry_transaction)get_configuration)format_new_turn_with_timestamp)PromptRepresentationRepresentation)track_deriver_input_tokens   )estimate_deriver_prompt_tokensminimal_deriver_promptreturnc                  $    t           j        j        S N)r   DERIVERMODEL_CONFIG     ;/DATA/AppData/hermes/projects/honcho/src/deriver/deriver.py_get_deriver_model_configr&       s    ((r$   minimal_deriver_batchderiver)opF)hit_batch_token_capwas_flush_enabledbatch_max_tokensmessagesmessage_level_configuration	observersobservedqueue_item_message_idsr*   r+   r,   c                N  %K   | sdS t          j                    }|                     d            | d         }	| d         }
|t          d          4 d{V }t	          dt          j        ||	j        |	j                   d{V t          j	        ||	j                   d{V           }ddd          d{V  n# 1 d{V swxY w Y   |j
        j        du rdS |j
        j        }t          d	|	j         d
 d|
j        d           t          d	|	j         d
 d|	j        d           d                    d | D                       }t!          |          }t#          |          %t%          %fd| D                       }t'          t(          j        t,          j        |t,          j        |i           t3          ||          }t          j                    |z
  dz  }t          d	|	j         d
 d|d           t5                      }|j        pt8          j        j        }|}t          j                    }t?          |||dt@          dt8          j!        j"        dddtG          |	j        tH          j%        j&        d                     d{V }t          j                    |z
  dz  }t          d	|	j         d
 d|d           t8          j'        j(        rJtS          j*        |j+        t(          j        j&        tX          j-        j&        t,          j.        j&                   fd| D             }t_          j0        |j1        ||	j        |	j2                  }d}|3                                s|s3th          5                    d |
j        |	j        |	j        |	j                   n~|D ]{}tm          |	j        |!          }	 |7                    |||	j        |	j2        |           d{V  |d"z  }I# tp          $ r&}th          9                    d#||           Y d}~td}~ww xY wt          j                    |z
  dz  }t          d	|	j         d
 d$|d           tu          |j;                  tu          |j<                  z   }t          d	|	j         d
 d%|d&           t8          j!        j=        r\t          d	|	j         d
 d'|d(           t          d	|	j         d
 d)d                    d* |j;        D                       d(           t}          d|	j         d
            tu          |          } tu          |           }!t%          d+ | D                       }"t          |!| z
  d          }#t          |"|z
  d          }$|j@        |k     r(th          5                    d,|j@        ||	jA                   |dk    r"th          5                    d-||	jA                   t          t          dHi d.|	j        d/|	j        d0d1tu          |          d2|
jA        d3|	jA        d4tu          |           d5tu          |j;                  d6|d7|d8|d9|d:|j@        d;|j+        d<| d=|!d>|"d?|#d@|$dA|dB|dCt8          j!        j"        dD|dE|dF|jD        dG|           dS )Ia  
    Process messages with minimal overhead - single LLM call, save to multiple collections.

    Args:
        messages: List of messages to process (includes interleaving context).
        message_level_configuration: Optional configuration override.
        observers: List of observer peer IDs (collections to save to).
        observed: The observed peer ID.
        queue_item_message_ids: Message IDs from queue items being processed
        hit_batch_token_cap: queue batcher clamped this batch to fit
        was_flush_enabled: DERIVER.FLUSH_ENABLED snapshot at batch time
        batch_max_tokens: DERIVER.REPRESENTATION_BATCH_MAX_TOKENS snapshot
    Nc                     | j         S r    )id)xs    r%   <lambda>z4process_representation_tasks_batch.<locals>.<lambda>B   s     r$   )keyr   zminimal_deriver.get_config)workspace_nameFminimal_deriver__starting_message_idr4   ending_message_id
c              3   V   K   | ]$}t          |j        |j        |j                  V  %d S r    )r   content
created_at	peer_name.0msgs     r%   	<genexpr>z5process_representation_tasks_batch.<locals>.<genexpr>i   sH       # # 	's{CNCMRR# # # # # #r$   c              3   8   K   | ]}|j         v |j        V  d S r    )r4   token_count)rD   rE   queue_item_message_ids_sets     r%   rF   z5process_representation_tasks_batch.<locals>.<genexpr>q   s;        sv9S/S/S/S/S/S/S r$   )	task_type
components)peer_idr-   custom_instructionsi  context_preparationmszMinimal DeriverT   minimal_deriverrepresentation)r9   call_purposeparent_categoryr0   )model_configprompt
max_tokens
track_nameresponse_model	json_modemax_input_tokensenable_retryretry_attempts
trace_name	telemetryllm_call_duration)countrJ   
token_type	componentc                 4    g | ]}|j         k    |j        S r#   )rB   r4   )rD   mr0   s     r%   
<listcomp>z6process_representation_tasks_batch.<locals>.<listcomp>   s'    EEEAQ[H-D-D14-D-D-Dr$   z@Deriver generated zero observations for messages %s:%s in %s/%s!)r9   observerr0   r   z1Failed to save representation for observer %s: %stotal_processing_timeobservation_countra   r-   blobexplicit_observationsc              3       K   | ]	}d | V  
dS )u    • Nr#   )rD   obss     r%   rF   z5process_representation_tasks_batch.<locals>.<genexpr>   s(      EEmcmmEEEEEEr$   c              3   $   K   | ]}|j         V  d S r    )rH   rC   s     r%   rF   z5process_representation_tasks_batch.<locals>.<genexpr>  s$      DDCDDDDDDr$   u   token-breakdown invariant violated: response.input_tokens (%d) < messages_tokens (%d) for observed=%s, latest=%s — provider tokenization drift or wrong messages_tokens computation?u}   prompt_scaffold_tokens estimated as %d for observed=%s, latest=%s — estimate_deriver_prompt_tokens may have failed silentlyr9   session_namer0   queue_items_processedearliest_message_idlatest_message_idmessage_countexplicit_conclusion_countcontext_preparation_msllm_call_mstotal_duration_msinput_tokenstotal_input_tokensoutput_tokensqueued_message_countprompt_message_countprompt_message_tokensextra_context_message_countextra_context_tokensprompt_scaffold_tokensr,   r[   r+   r*   hit_input_token_capobserver_countr#   )Etimeperf_countersortr   r   r   get_sessionro   r9   get_workspace	reasoningenabledrM   r   r4   joinr   setsumr   r   	INGESTIONr   PROMPTMESSAGESr   r&   max_output_tokensr   LLMDEFAULT_MAX_TOKENSr   r   r!   MAX_INPUT_TOKENSr	   r   DERIVER_REPRESENTATIONvalueMETRICSENABLEDr   record_deriver_tokensrz   r   OUTPUTOUTPUT_TOTALr   from_prompt_representationr@   rA   is_emptyloggerwarningr   save_representation	Exceptionerrorlenexplicit	deductiveLOG_OBSERVATIONSr   maxrx   	public_idr   r   r   )&r-   r.   r/   r0   r1   r*   r+   r,   overall_startlatest_messageearliest_messagedbrM   formatted_messagesprompt_tokensmessages_tokensrV   context_prep_durationbase_model_configrW   rU   	llm_startresponsellm_durationmessage_idsobservationssuccessful_observer_countrg   representation_managereoverall_durationtotal_observationsr{   r|   r}   r~   r   rI   s&      `                                 @r%   "process_representation_tasks_batchr   $   s	     2  %''MMMnnM%%%b\N{
 #*:;; 		 		 		 		 		 		 		r*;&3^5R        (~'D        + +'		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 #,4==5?S9>,99x99	   9>,99x99	    # ## # #   33FGGM!$%;!<!<    #+    O ",$m&
    $#/  F ".00=@DH9>,99x99	   233"4W8WJ$L !##I$!$+!):$%)8$;A,	
 
 
        H$ %'')3t;L9>,99x99	    
0(&06!(.'4:		
 	
 	
 	
 FEEEEEEK "<#!	 L !" k N)'	
 	
 	
 	
 " 	 	H%:-<!!& & &",@@ "/"-/         *Q.))   GST        )++m;tC9>,99x99	   \233c,:P6Q6QQ9>,99x99	   ( 
=~0==8==		
 	
 	
 	=~0==8==#IIEE|/DEEEEE		
 	
 	
 -.2C/P/Ph/P/PQQQ 566x==DD8DDDDD"%&:=Q&QST"U"U4FJJ .. E!$	
 	
 	
  L$		
 	
 	
 	$ 	
 	
 	
)88	
'44	
 X	
 #&&<"="="=		

 !1 : :	
 -66	
 h---	
 '*,*?&@&@&@	
 $9#8	
 %	
 /.	
 )	
  (44	
 #00	
  "6!5!	
" "6!5#	
$ #8"7%	
& )D(C'	
( "6!5)	
* $1=+	
, .--	
. &->>/	
0 0/1	
2 !4 33	
4 !) < <5	
6 547	
    s+   AC
CC.O44
P$>PP$)8loggingr   srcr   
src.configr   r   src.crud.representationr   src.dependenciesr   src.llmr   src.llm.typesr	   
src.modelsr
   src.schemasr   src.telemetryr   src.telemetry.eventsr   r   src.telemetry.events.llmr   src.telemetry.loggingr   r    src.telemetry.prometheus.metricsr   r   r   src.telemetry.sentryr   src.utils.config_helpersr   src.utils.formattingr   src.utils.representationr   r   src.utils.tokensr   promptsr   r   	getLogger__name__r   r&   liststrintboolr   r#   r$   r%   <module>r      s           8 8 8 8 8 8 8 8 9 9 9 9 9 9 ' ' ' ' ' ' # # # # # # - - - - - -       - - - - - - , , , , , , C C C C C C C C 0 0 0 0 0 0 L L L L L L L L         
 9 8 8 8 8 8 6 6 6 6 6 6 ? ? ? ? ? ? I I I I I I I I 7 7 7 7 7 7 K K K K K K K K		8	$	$)#: ) ) ) ) 0Y??? !&#X X X7mX!6!=X Cy	X
 X !IX X X X 
X X X @?X X Xr$   