Ë
    õ¡ŸjC  ã                  óÀ   — d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZ  ed¬«      dd	„«       Z e
 ed
dd¬«       ed«      ¬«      dd„«       Zdd„Zy)zM
author: Yagnik Poshiya
organization: Webbrains Technologies Private Limited
é    )Úannotations)Ú	lru_cache)ÚList)Úgenai)ÚretryÚstop_after_attemptÚwait_exponentialé   )Úmaxsizec                ó.   — t        j                  | ¬«      S )N©Úapi_key)r   ÚClientr   s    úk/var/www/html/agent-engine/apps/workers/async/embedding-pipeline-worker/src/tokenization/local_tokenizer.pyÚ_get_clientr      s   € ô �<‰< Ô(Ð(ó    g      à?é   é   )Ú
multiplierÚminÚmaxé   )ÚwaitÚstopc                óª   — t        | «      }|j                  j                  ||¬«      }|j                  €t	        d«      ‚t        |j                  «      S )N)ÚmodelÚcontentsz%count_tokens returned no total_tokens)r   ÚmodelsÚcount_tokensÚtotal_tokensÚRuntimeErrorÚint)r   Ú
model_nameÚtextÚclientÚresults        r   Ú
_count_oner'      sN   € ä˜Ó!€FØ�]‰]×'Ñ'¨jÀ4Ð'ÓH€FØ×ÑÐ"äÐBÓCÐCÜˆv×"Ñ"Ó#Ð#r   c                óD   — |D �cg c]  }t        | ||¬«      ‘Œ c}S c c}w )N)r   r#   r$   )r'   )r   r#   ÚtextsÚts       r   Úcount_tokens_for_textsr+   !   s#   € ð QVÖVÈ1ŒJ˜w°:ÀAÖFÒVÐVùÒVs   …N)r   ÚstrÚreturnzgenai.Client)r   r,   r#   r,   r$   r,   r-   r"   )r   r,   r#   r,   r)   z	List[str]r-   z	List[int])Ú__doc__Ú
__future__r   Ú	functoolsr   Útypingr   Úgoogler   Útenacityr   r   r	   r   r'   r+   © r   r   ú<module>r5      sm   ðñõ
 #å "å Ý å ß @Ñ @ñ �1Ôò)ó ð)ñ
 Ñ¨°¸Ô;ÑBTÐUVÓBWÔXò$ó Yð$ôWr   