diff --git a/reflexio/server/services/service_utils.py b/reflexio/server/services/service_utils.py index e1679aa1..e3672dca 100644 --- a/reflexio/server/services/service_utils.py +++ b/reflexio/server/services/service_utils.py @@ -120,7 +120,8 @@ def slice_content_by_tokens(content: str, max_tokens: int | None) -> str: // 2`` tokens and the last ``max_tokens - max_tokens // 2`` tokens, joined by a truncation marker so the elision is visible to the LLM. Content within budget (or when ``max_tokens`` is None / content is empty) is returned - unchanged. + unchanged. Literal tokenizer markers in user content are encoded as ordinary + text rather than interpreted as special tokens. Args: content (str): The interaction content to slice. @@ -132,7 +133,7 @@ def slice_content_by_tokens(content: str, max_tokens: int | None) -> str: if max_tokens is None or not content: return content encoding = _get_content_token_encoding() - tokens = encoding.encode(content) + tokens = encoding.encode(content, disallowed_special=()) if len(tokens) <= max_tokens: return content head = max_tokens // 2 diff --git a/tests/e2e_tests/test_profile_workflows.py b/tests/e2e_tests/test_profile_workflows.py index 2fe48d07..414703f1 100644 --- a/tests/e2e_tests/test_profile_workflows.py +++ b/tests/e2e_tests/test_profile_workflows.py @@ -42,14 +42,23 @@ @skip_in_precommit +@pytest.mark.parametrize("literal_token", ["", "<|endoftext|>"]) def test_publish_interaction_profile_only( reflexio_instance_profile_only: Reflexio, sample_interaction_requests: list[InteractionData], cleanup_profile_only: Callable[[], None], + literal_token: str, + monkeypatch: pytest.MonkeyPatch, ): """Test interaction publishing with only profile extraction enabled.""" + # Exercise real prompt construction; the E2E fixture still mocks LiteLLM. + monkeypatch.setenv("MOCK_LLM_RESPONSE", "false") user_id = "test_user_profile_only" agent_version = "test_agent_profile" + if literal_token: + sample_interaction_requests[0] = sample_interaction_requests[0].model_copy( + update={"content": sample_interaction_requests[0].content + literal_token} + ) # Publish interactions (request_id will be auto-generated) response = reflexio_instance_profile_only.publish_interaction( diff --git a/tests/server/services/test_service_utils.py b/tests/server/services/test_service_utils.py index 80fd3962..eb7494fd 100644 --- a/tests/server/services/test_service_utils.py +++ b/tests/server/services/test_service_utils.py @@ -442,9 +442,16 @@ def test_format_sessions_to_history_string_preserves_order_within_group(): assert result == expected -def test_slice_content_by_tokens_within_budget_unchanged(): +@pytest.mark.parametrize( + "content", + [ + "short content well under budget", + "Literal <|endoftext|> <|fim_prefix|> <|fim_middle|> " + "<|fim_suffix|> <|endofprompt|> in user text", + ], +) +def test_slice_content_by_tokens_within_budget_unchanged(content): """Content at or below the budget is returned verbatim.""" - content = "short content well under budget" assert slice_content_by_tokens(content, 512) == content @@ -459,11 +466,12 @@ def test_slice_content_by_tokens_empty_content(): assert slice_content_by_tokens("", 512) == "" -def test_slice_content_by_tokens_keeps_head_and_tail(): +@pytest.mark.parametrize("prefix", ["", "Literal <|endoftext|> <|fim_prefix|> "]) +def test_slice_content_by_tokens_keeps_head_and_tail(prefix): """Over-budget content keeps the first half + last half with a marker.""" encoding = _get_content_token_encoding() - content = " ".join(str(i) for i in range(2000)) - tokens = encoding.encode(content) + content = prefix + " ".join(str(i) for i in range(2000)) + tokens = encoding.encode_ordinary(content) assert len(tokens) > 512 # precondition: actually over budget result = slice_content_by_tokens(content, 512) @@ -474,7 +482,28 @@ def test_slice_content_by_tokens_keeps_head_and_tail(): assert result == expected assert _CONTENT_TRUNCATION_MARKER in result # The sliced result is materially shorter than the original. - assert len(encoding.encode(result)) < len(tokens) + assert len(encoding.encode_ordinary(result)) < len(tokens) + + +@pytest.mark.parametrize("max_tokens", [16, 512]) +def test_literal_token_markers_in_prompt_and_evidence(monkeypatch, max_tokens): + """Prompt text and grounded evidence share literal-safe content slicing.""" + monkeypatch.setenv(_ENV_MAX_TOKENS, str(max_tokens)) + content = "<|endoftext|> " + "ordinary text " * 30 + "<|fim_prefix|>" + interaction = _create_interaction(1, content, "user", 1_700_000_000) + encoding = _get_content_token_encoding() + tokens = encoding.encode_ordinary(content) + expected_parts = ( + (encoding.decode(tokens[:8]), encoding.decode(tokens[-8:])) + if max_tokens == 16 + else (content,) + ) + + history = format_interactions_to_history_string([interaction]) + evidence = visible_interaction_evidence_texts(interaction) + + assert history == f"user: ```{_CONTENT_TRUNCATION_MARKER.join(expected_parts)}```" + assert evidence == expected_parts def test_resolve_max_tokens_unset_uses_default(monkeypatch):