Ë
    õ¡Ÿj½  ã                  óp   — d Z ddlmZ ddlZddlmZ  ej                  d«      Zdd„Z	 	 	 	 	 	 	 	 	 	 d	d„Z	y)
zM
author: Yagnik Poshiya
organization: Webbrains Technologies Private Limited
é    )ÚannotationsN)Úsanitize_text_for_dbz\s+c                óŒ   — t        | xs d«      }|j                  «       }t        j                  d|«      }|j                  «       S )NÚ ú )r   ÚstripÚ_WHITESPACE_REÚsub)ÚtextÚts     ú^/var/www/html/agent-engine/apps/workers/async/embedding-pipeline-worker/src/chunking/simple.pyÚ_clean_textr      s9   € Ü˜TšZ RÓ(€AØ	�‰‹	€AÜ×Ñ˜3 Ó"€AØ�7‰7‹9Ðó    c               ó>  ‡‡‡— | xs dj                  «       }|sg S t        j                  d|«      D �cg c]&  }|sŒ|j                  «       sŒ|j                  «       ‘Œ( }}|s|g}g Šg ŠdŠˆˆˆfd„}|D �]  }t        |«      |kD  rš |«        d}|t        |«      k  r‚t        ‰«      |k  rtt	        t        |«      ||z   «      }	t        |||	 «      }
|
r‰j                  |
«       |	t        |«      k\  rn,t        d|	|z
  «      }|t        |«      k  rt        ‰«      |k  rŒtŒ¬‰t        |«      z   dz   |k  r#‰j                  |«       ‰t        |«      dz   z  ŠŒã |«        ‰j                  |«       t        |«      Št        ‰«      |k\  s�Œ n  |«        ‰d| S c c}w )zx
    Simple character-window chunking with overlap.

    This is intentionally tokenization-free for an MVP worker.
    r   z\n\s*\nr   c                 ón   •— ‰sy t        dj                  ‰«      «      } | r‰j                  | «       g ŠdŠy )Nz

r   )r   ÚjoinÚappend)ÚjoinedÚbufÚbuf_lenÚchunkss    €€€r   Ú	flush_bufzchunk_text.<locals>.flush_buf0   s5   ø€ áØÜ˜VŸ[™[¨Ó-Ó.ˆÙØ�M‰M˜&Ô!ØˆØ‰r   é   N)r   ÚreÚsplitÚlenÚminr   r   Úmax)r   Ú	max_charsÚoverlap_charsÚ
max_chunksr   ÚpÚ
paragraphsr   ÚstartÚendÚwindowr   r   r   s              @@@r   Ú
chunk_textr'      s“  ú€ ð 
Š�×ÑÓ€AÙØˆ	ô &(§X¡X¨j¸!Ó%<ÖP ÂÀaÇgÁgÅi�!—'‘'•)ÐP€JÐPÙØ�Sˆ
à€FØ€CØ€Göð ó ˆÜˆq‹6�IÒÙŒKØˆEØœ#˜a›&’.¤S¨£[°:Ò%=Üœ#˜a›& %¨)Ñ"3Ó4�Ü$ Q u¨S \Ó2�ÙØ—M‘M &Ô)Øœ#˜a›&’=ØÜ˜A˜s ]Ñ2Ó3�ð œ#˜a›&’.¤S¨£[°:Ó%=ð à”S˜“VÑ˜aÑ 9Ò,Ø�J‰J�qŒMØ”s˜1“v ‘zÑ!ˆGØáŒØ�
‰
�1ŒÜ�a“&ˆäˆv‹;˜*Ô$Ùð1ñ4 „Kà�+�:ÐÐùò] Qs   ´F¼FÁF)r   ÚstrÚreturnr(   )
r   r(   r   Úintr    r*   r!   r*   r)   z	list[str])
Ú__doc__Ú
__future__r   r   Úutils.text_sanitizer   Úcompiler	   r   r'   © r   r   ú<module>r0      sa   ðñõ
 #ã 	å 4ð �—‘˜FÓ#€óð?Ø
ð?ð ð?ð ð	?ð
 ð?ð ô?r   