core/tools: add truncate token module (chainable cursors)

tests/test_tools/test_truncate_token.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from plyngent.tools.truncate_token import (
+    TruncateToken,
+    decode_truncate_token,
+    encode_truncate_token,
+    truncate_with_token,
+)
+
+
+def test_token_roundtrip() -> None:
+    token = TruncateToken(kind="file", location="src/a.py", offset=100, limit=32_000)
+    encoded = encode_truncate_token(token)
+    assert decode_truncate_token(encoded) == token
+
+
+def test_decode_garbage_returns_none() -> None:
+    assert decode_truncate_token("not-a-token") is None
+    assert decode_truncate_token("") is None
+    assert decode_truncate_token("!!!") is None
+
+
+def test_decode_rejects_invalid_fields() -> None:
+    bad = TruncateToken(kind="file", location="x", offset=-1, limit=10)
+    assert decode_truncate_token(encode_truncate_token(bad)) is None
+    bad_limit = TruncateToken(kind="http", location="u", offset=0, limit=0)
+    assert decode_truncate_token(encode_truncate_token(bad_limit)) is None
+
+
+def test_truncate_with_token_short_text_untouched() -> None:
+    out, token = truncate_with_token(
+        "hello",
+        32_000,
+        kind="file",
+        location="a.txt",
+        offset=0,
+        limit=32_000,
+        total_len=5,
+    )
+    assert out == "hello"
+    assert token is None
+
+
+def test_truncate_with_token_cuts_and_advances() -> None:
+    text = "x" * 500
+    out, token = truncate_with_token(
+        text,
+        200,
+        kind="file",
+        location="a.txt",
+        offset=0,
+        limit=200,
+        total_len=500,
+    )
+    assert token is not None
+    assert "truncated 300 chars" in out
+    assert f"[TRUNCATE_TOKEN: {encode_truncate_token(token)}]" in out
+    assert token.offset == 200
+    assert token.limit == 200
+    assert len(out) <= 200  # marker fits inside the char budget
+
+
+def test_truncate_with_token_more_after_nonoverflow() -> None:
+    # Segment fits the chunk, but the source continues: still emit a token.
+    text = "y" * 30
+    out, token = truncate_with_token(
+        text,
+        40,
+        kind="file",
+        location="a.txt",
+        offset=0,
+        limit=40,
+        total_len=200,
+    )
+    assert token is not None
+    assert token.offset == 40
+    assert "more content available" in out
+    assert out.startswith("y" * 30)
+
+
+def test_truncate_with_token_end_reached_no_token() -> None:
+    text = "z" * 30
+    out, token = truncate_with_token(
+        text,
+        40,
+        kind="http",
+        location="https://example.com/x",
+        offset=0,
+        limit=40,
+        total_len=30,
+    )
+    assert token is None
+    assert out == text
+
+
+def test_truncate_with_token_nonzero_start() -> None:
+    text = "y" * 500
+    out, token = truncate_with_token(
+        text,
+        200,
+        kind="http",
+        location="https://example.com/x",
+        offset=150,
+        limit=200,
+        total_len=700,
+    )
+    assert token is not None
+    assert token.kind == "http"
+    assert token.offset == 350  # 150 + 200
+    assert "truncated 300 chars" in out
+    assert len(out) <= 200