Ë
    õ¡Ÿji"  ã                  ó@  — d Z ddlmZ ddlZddlmZ ddlmZmZmZ ddl	m
Z
mZ ddlZddlmZ dd„Zdd	„Z ed
¬«       G d„ d«      «       Z	 	 	 	 	 	 dd„Zddœ	 	 	 	 	 dd„Zddœ	 	 	 	 	 	 	 dd„Zddœ	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 dd„Zddœ	 	 	 	 	 	 	 	 	 	 	 	 	 d d„Zy)!zM
author: Yagnik Poshiya
organization: Webbrains Technologies Private Limited
é    )ÚannotationsN)Ú	dataclass)ÚdatetimeÚ	timedeltaÚtimezone)ÚAnyÚOptional)ÚClientc                 óH   — t        j                  t        j                  «      S ©N)r   Únowr   Úutc© ó    úY/var/www/html/agent-engine/apps/workers/async/embedding-pipeline-worker/src/queue/jobs.pyÚ_utcnowr      s   € Ü�<‰<œŸ™Ó%Ð%r   c                 ó*   — t        j                  «       S r   )ÚsocketÚgethostnamer   r   r   Ú
_worker_idr      s   € ä×ÑÓÐr   T)Úfrozenc                  ó|   — e Zd ZU ded<   ded<   ded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   y)ÚDocumentProcessingJobÚstrÚidÚorganization_idÚ
project_idÚdocument_idÚstatusÚjob_typeÚintÚpriorityzdict[str, Any] | NoneÚpayloadÚattempt_countÚmax_attemptsúOptional[str]Ú	run_afterN)Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r   r   r   r   r      s?   … àƒGØÓØƒOØÓØƒKØƒMØƒMØ"Ó"ØÓØÓØÔr   r   c               óÈ  — t        j                  d«      }t        «       j                  «       }| j	                  d«      j                  d«      j                  dd«      j                  d|«      j                  dd¬	«      j                  d
d¬	«      j                  |«      j                  «       }|j                  xs g }|j                  dt        |«      «       g }|D �]  }|j                  t        t!        |d   «      t!        |d   «      t!        |d   «      t!        |d   «      t!        |j#                  d«      xs d«      t!        |j#                  d«      xs d«      t%        |j#                  d«      xs d«      |j#                  d«      t%        |j#                  d«      xs d«      t%        |j#                  d«      xs d«      |j#                  d«      ¬«      «       �Œ |S )Núembedding-worker.queueÚdocument_processing_jobszoid,organization_id,project_id,document_id,status,job_type,priority,payload,attempt_count,max_attempts,run_afterr   Úqueuedr'   r"   T©ÚdescÚ
created_atFzlist_queued_jobs rows=%sr   r   r   r   Ú r    r   r#   r$   r%   )r   r   r   r   r   r    r"   r#   r$   r%   r'   )ÚloggingÚ	getLoggerr   Ú	isoformatÚtableÚselectÚeqÚlteÚorderÚlimitÚexecuteÚdataÚdebugÚlenÚappendr   r   Úgetr!   )Úsupabaser<   ÚlogÚnow_isoÚresÚrowsÚjobsÚrs           r   Úlist_queued_jobsrJ   )   s”  € ô
 ×
Ñ
Ð4Ó
5€Cô ‹i×!Ñ!Ó#€Gà�‰Ð1Ó2ß	‰Ø}ó

÷ 
‰ˆH�hÓ	ß	‰ˆ[˜'Ó	"ß	‰ˆz ˆÓ	%ß	‰ˆ| %ˆÓ	(ß	‰ˆu‹ß	‰‹ð ð �8‰8Š>�r€DØ‡I�IÐ(¬#¨d«)Ô4Ø(*€DØó 
ˆØ�‰Ü!Ü�q˜‘w“<Ü # AÐ&7Ñ$8Ó 9Ü˜q ™Ó/Ü  -Ñ 0Ó1Ü˜1Ÿ5™5 ›?Ò0¨bÓ1Ü˜QŸU™U :Ó.Ò4°"Ó5Ü˜QŸU™U :Ó.Ò3°!Ó4ØŸ™˜iÓ(Ü! !§%¡%¨Ó"8Ò"=¸AÓ>Ü  §¡ ~Ó!6Ò!;¸!Ó<ØŸ%™% Ó,ôö	
ð
ð  €Kr   éÈ   )r<   c               óD  — t        j                  d«      }t        «       j                  «       }| j	                  d«      j                  d«      j                  dd«      j                  j                  dd«      j                  d|«      j                  dd¬	«      j                  |«      j                  «       }|j                  xs g }|sy
d
}|D ]¼  }t        |j                  d«      xs d
«      }t        |j                  d«      xs d
«      }	|	d
k  s||	k\  rŒJt!        |d   «      }
| j	                  d«      j#                  dddddœ«      j                  d|
«      j                  dd«      j                  «       }|j                  sŒ¸|dz  }Œ¾ |r|j%                  d|«       |S )z‘
    Move failed jobs back to queued when their retry time is reached.
    This keeps the claim path simple: worker only claims queued jobs.
    r-   r.   z'id,attempt_count,max_attempts,run_afterr   Úfailedr'   ÚnullFr0   r   r$   r%   r   r/   N)r   Úlease_expires_atÚ	locked_atÚ	locked_byé   zrequeued_failed_jobs count=%s)r4   r5   r   r6   r7   r8   r9   Únot_Úis_r:   r;   r<   r=   r>   r!   rB   r   ÚupdateÚinfo)rC   r<   rD   rE   rF   rG   ÚrequeuedrI   r$   r%   ÚjidÚupds               r   Úrequeue_retryable_failed_jobsrZ   T   sr  € ô ×
Ñ
Ð4Ó
5€CÜ‹i×!Ñ!Ó#€Gð 	�‰Ð1Ó2ß	‰Ð9Ó	:ß	‰ˆH�hÓ	ß	‰�c‰c�+˜vÓ&ß	‰ˆ[˜'Ó	"ß	‰ˆ{ ˆÓ	'ß	‰ˆu‹ß	‰‹ð ð �8‰8Š>�r€DÙØà€HØò ˆÜ˜AŸE™E /Ó2Ò7°aÓ8ˆÜ˜1Ÿ5™5 Ó0Ò5°AÓ6ˆØ˜1Ò °Ò =ØÜ�!�D‘'‹lˆà�N‰NÐ5Ó6ß‰V˜x¸TÐPTÐcgÑhÓiß‰R��c‹]ß‰R�˜(Ó#ß‰W‹Yð 	ð �8‹8Ø˜‰M‰Hðñ  Ø�‰Ð0°(Ô;Ø€Or   c          
     ó   — t        j                  d«      }t        «       }|j                  «       }| j	                  d«      j                  d«      j                  dd«      j                  j                  dd«      j                  d|«      j                  dd¬	«      j                  |«      j                  «       }|j                  xs g }|sy
d
}|D �]  }	t        |	d   «      }
t        |	j!                  d«      xs d
«      }t        |	j!                  d«      xs d
«      }|dz   }|d
kD  xr ||k\  }|dddddœ}|r|j#                  dd|dœ«       n8|j#                  d|t%        t        |«      ¬«      z   j                  «       dœ«       | j	                  d«      j#                  |«      j                  d|
«      j                  dd«      j                  «       }|j                  s�Œ|dz  }�Œ |r|j'                  d|«       |S )a6  
    Reclaim jobs stuck in `processing` after lease expiry.

    If a worker crashes mid-job, the job can remain `processing` forever.
    This function:
    - increments attempt_count
    - sets status to `failed` if max_attempts reached
    - otherwise re-queues the job with run_after = now() + backoff
    r-   r.   zid,attempt_count,max_attemptsr   Ú
processingrO   rN   Fr0   r   r   r$   r%   rR   z6Lease expired while processing (worker crash/timeout).N)r$   Ú
last_errorrO   rP   rQ   rM   )r   r'   Úcompleted_atr/   ©Úseconds)r   r'   zreaped_expired_leases count=%s)r4   r5   r   r6   r7   r8   r9   rS   rT   r:   r;   r<   r=   r>   r   r!   rB   rU   r   Úwarning)rC   Úbackoff_secondsr<   rD   r   rE   rF   rG   Ú	reclaimedrI   rX   r$   r%   Únext_attemptÚterminalrU   rY   s                    r   Úreap_expired_processing_leasesrf   …   sé  € ô ×
Ñ
Ð4Ó
5€CÜ
‹)€CØ�m‰m‹o€Gð 	�‰Ð1Ó2ß	‰Ð/Ó	0ß	‰ˆH�lÓ	#ß	‰�c‰cÐ$ fÓ-ß	‰Ð Ó	)ß	‰Ð!¨ˆÓ	.ß	‰ˆu‹ß	‰‹ð ð �8‰8Š>�r€DÙØà€IØó !ˆÜ�!�D‘'‹lˆÜ˜AŸE™E /Ó2Ò7°aÓ8ˆÜ˜1Ÿ5™5 Ó0Ò5°AÓ6ˆØ$ qÑ(ˆØ !Ñ#ÒD¨¸Ñ(Dˆð *ØRØ $ØØñ
ˆñ Ø�M‰Mà&Ø!%Ø$+ñõð �M‰Mà&Ø"%¬	¼#¸oÓ:NÔ(OÑ"O×!ZÑ!ZÓ!\ñôð �n‰nÐ7Ó8×?Ñ?ÀÓG×JÑJÈ4ÐQTÓU×XÑXÐYaÐcoÓp×xÑxÓzˆØ�8Œ8Ø˜‰NŠIðC!ñF Ø�‰Ð4°iÔ@ØÐr   )Ú	worker_idc               ó  — t        j                  d«      }|xs
 t        «       }t        «       }|t	        t        |«      ¬«      z   }| j                  d«      j                  d|j                  «       ||j                  «       |j                  «       dœ«      j                  d|«      j                  dd«      j                  «       }|j                  xs g }	|	r"|j                  d	|||j                  «       «       |	r|	d
   S d S )Nr-   r_   r.   r\   )r   rP   rQ   rO   Ú
started_atr   r   r/   z2claimed job_id=%s locked_by=%s lease_expires_at=%sr   )r4   r5   r   r   r   r!   r7   rU   r6   r9   r=   r>   rV   )
rC   Újob_idÚlease_secondsrg   rD   Úwidr   rO   rF   rG   s
             r   Útry_claim_jobrm   Ï   så   € ô ×
Ñ
Ð4Ó
5€CØ
Ò
#”z“|€CÜ
‹)€CØœY¬s°=Ó/AÔBÑBÐð 	�‰Ð1Ó2ß	‰à&Ø Ÿ]™]›_Ø Ø$4×$>Ñ$>Ó$@Ø!Ÿm™m›oñó

÷ 
‰ˆD�&Ó	ß	‰ˆH�hÓ	ß	‰‹ð ð �8‰8Š>�r€DÙØ�‰ÐEÀvÈsÐTd×TnÑTnÓTpÔqÙˆ4�‰7Ð$ Ð$r   c               ó  — t        j                  d«      }t        «       j                  «       }| j	                  d«      j                  d|d dœ«      j                  d|«      j                  «        |j                  d|«       y )Nr-   r.   Ú	completed)r   r^   rO   r   zcompleted job_id=%s)	r4   r5   r   r6   r7   rU   r9   r=   rV   )rC   rj   rD   r   s       r   Úcomplete_jobrp   ô   sn   € ô
 ×
Ñ
Ð4Ó
5€CÜ
‹)×
Ñ
Ó
€CØ‡N�NÐ-Ó.×5Ñ5à!ØØ $ñ	
ó÷ 	�bˆˆvÓ—w‘w”yØ‡H�HÐ" FÕ+r   é<   )rb   c          	     ó¸  — t        j                  d«      }t        «       }|t        t	        |«      ¬«      z   }|dz   }	|	|k\  }
| j                  d«      j                  d|	|d d |
rd n|j                  «       d |
r|j                  «       nd dœ«      j                  d|«      j                  «        |j                  d	||
|	||
rd «       y |j                  «       «       y )
Nr-   r_   rR   r.   rM   i   )r   r$   r]   r'   rO   r^   r   z6failed job_id=%s terminal=%s attempt=%s/%s retry_at=%s)r4   r5   r   r   r!   r7   rU   r6   r9   r=   ra   )rC   rj   Úerrorr$   r%   rb   rD   r   Únext_runrd   re   s              r   Úfail_jobru     sÙ   € ô ×
Ñ
Ð4Ó
5€CÜ
‹)€CØ”Y¤s¨?Ó';Ô<Ñ<€HØ  1Ñ$€LØ˜|Ñ+€Hà‡N�NÐ-Ó.×5Ñ5ð Ø)Ø  ˜,Ù!)™¨x×/AÑ/AÓ/CØ $Ù/7˜CŸM™MœO¸Tñ	
ó
÷ 	�bˆˆvÓ—w‘w”yØ‡K�KØ@ØØØØÙˆõð '×0Ñ0Ó2õr   )Úreturnr   )rv   r   )rC   r
   r<   r!   rv   zlist[DocumentProcessingJob])rC   r
   r<   r!   rv   r!   )rC   r
   rb   r!   r<   r!   rv   r!   )
rC   r
   rj   r   rk   r!   rg   r&   rv   zOptional[dict[str, Any]])rC   r
   rj   r   rv   ÚNone)rC   r
   rj   r   rs   r   r$   r!   r%   r!   rb   r!   rv   rw   )Ú__doc__Ú
__future__r   r   Údataclassesr   r   r   r   Útypingr   r	   r4   Úsupabase.clientr
   r   r   r   rJ   rZ   rf   rm   rp   ru   r   r   r   ú<module>r}      su  ðñõ
 #ã Ý !ß 2Ñ 2ß  ã Ý "ó&ó ñ
 �$Ô÷ð ó ðð(Øð(ð ð(ð !ó	(ð\ ñ.Øð.ð ð.ð 	ó	.ðj ñ	GØðGð ðGð ð	Gð
 	óGð^  $ñ"%Øð"%ð ð"%ð ð	"%ð
 ð"%ð ó"%ðJ,Øð,ð ð,ð 
ó	,ð0 ñ!Øð!ð ð!ð ð	!ð
 ð!ð ð!ð ð!ð 
ô!r   