Skip to content

symfonic.core.documents.truncation

truncation

Token-aware text truncation utilities.

estimate_tokens 

estimate_tokens(text: str, chars_per_token: int = 4) -> int

Estimate token count from character count.

Source code in src/symfonic/core/documents/truncation.py 
19
20
21
def estimate_tokens(text: str, chars_per_token: int = 4) -> int:
    """Estimate token count from character count."""
    return len(text) // chars_per_token

truncate_to_tokens 

truncate_to_tokens(
    text: str, max_tokens: int, chars_per_token: int = 4
) -> str

Truncate text to fit within a token budget.

Uses a simple character-based estimate: max_tokens * chars_per_token.

Source code in src/symfonic/core/documents/truncation.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
def truncate_to_tokens(
    text: str, max_tokens: int, chars_per_token: int = 4
) -> str:
    """Truncate text to fit within a token budget.

    Uses a simple character-based estimate: ``max_tokens * chars_per_token``.
    """
    max_chars = max_tokens * chars_per_token
    if len(text) <= max_chars:
        return text
    return text[:max_chars] + "..."