diff --git a/examples/agent_client.py b/examples/agent_client.py
new file mode 100644
index 000000000..4c71e82fd
--- /dev/null
+++ b/examples/agent_client.py
@@ -0,0 +1,142 @@
+"""
+Minimal agent client for the InfiniLM inference server.
+
+Demonstrates the complete tool-call loop over the OpenAI-compatible
+``/v1/chat/completions`` endpoint:
+
+ request (with tools) -> model returns tool_calls
+ -> client executes the tools locally
+ -> results are appended as assistant/tool messages
+ -> repeat until the model answers in plain text
+
+This is the same loop an agent framework (e.g. Claude Code) drives; it is
+kept dependency-free and with a short system prompt on purpose, so the loop
+can be validated on small models where huge agent prompts degrade tool use.
+
+Usage (server started with ``--tool-call-parser llama31`` or
+``--tool-call-parser glm4-9b-0414`` etc.):
+
+ python examples/agent_client.py --url http://127.0.0.1:8000 \
+ --model GLM-4-9B-0414 "北京天气怎么样？顺便看看当前目录有什么文件"
+"""
+
+import argparse
+import json
+import os
+import urllib.request
+
+TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get the current weather for a city.",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string", "description": "City name"}},
+ "required": ["city"],
+ },
+ },
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "list_dir",
+ "description": "List files and directories at a path.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "path": {"type": "string", "description": "Directory path"}
+ },
+ "required": ["path"],
+ },
+ },
+ },
+]
+
+
+def execute_tool(name: str, arguments: dict) -> str:
+ """Execute a tool locally and return its result as a string."""
+ if name == "get_weather":
+ city = arguments.get("city", "")
+ return json.dumps(
+ {"city": city, "weather": "晴", "temperature": "26度"}, ensure_ascii=False
+ )
+ if name == "list_dir":
+ path = arguments.get("path", ".")
+ try:
+ entries = sorted(os.listdir(path))
+ except OSError as e:
+ return f"error: {e}"
+ return "\n".join(entries) if entries else "(empty directory)"
+ return f"error: unknown tool {name}"
+
+
+def chat(url: str, payload: dict) -> dict:
+ request = urllib.request.Request(
+ f"{url}/v1/chat/completions",
+ data=json.dumps(payload).encode(),
+ headers={"Content-Type": "application/json"},
+ )
+ with urllib.request.urlopen(request, timeout=600) as response:
+ return json.loads(response.read())
+
+
+def run(url: str, model: str, question: str, max_turns: int = 6):
+ messages = [
+ {
+ "role": "system",
+ "content": "You are a helpful assistant. Use the available tools "
+ "to answer the user's question.",
+ },
+ {"role": "user", "content": question},
+ ]
+
+ for turn in range(max_turns):
+ response = chat(
+ url,
+ {
+ "model": model,
+ "messages": messages,
+ "tools": TOOLS,
+ "max_tokens": 1024,
+ "stream": False,
+ },
+ )
+ choice = response["choices"][0]
+ message = choice["message"]
+ tool_calls = message.get("tool_calls") or []
+
+ if not tool_calls:
+ print(f"\n[turn {turn}] assistant: {message.get('content', '')}")
+ return
+
+ # Record the assistant tool-call turn, execute, feed results back.
+ messages.append(message)
+ for call in tool_calls:
+ name = call["function"]["name"]
+ try:
+ arguments = json.loads(call["function"]["arguments"] or "{}")
+ except json.JSONDecodeError:
+ arguments = {}
+ result = execute_tool(name, arguments)
+ print(f"[turn {turn}] tool_call: {name}({arguments}) -> {result[:80]}")
+ messages.append(
+ {"role": "tool", "tool_call_id": call["id"], "content": result}
+ )
+
+ print("\n[max turns reached without a final answer]")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--url", default="http://127.0.0.1:8000")
+ parser.add_argument("--model", default="GLM-4-9B-0414")
+ parser.add_argument("--max-turns", type=int, default=6)
+ parser.add_argument("question", nargs="+")
+ args = parser.parse_args()
+ run(args.url, args.model, " ".join(args.question), args.max_turns)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/pyproject.toml b/pyproject.toml
index 3ee4742c3..000afe0e8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,12 @@ name = "InfiniLM"
version = "0.1.0"
description = "InfiniLM model implementations"
readme = "README.md"
-dependencies = []
+dependencies = [
+ # Incremental (streaming) parsing of incomplete tool-call arguments.
+ # Without it the fallback decoder can only parse fully-formed JSON, so
+ # streamed tool-call arguments are held back until the object is complete.
+ "partial-json-parser>=0.2.1.1",
+]
requires-python = ">=3.10"
classifiers = [
"Programming Language :: Python :: 3",
diff --git a/python/infinilm/agents/__init__.py b/python/infinilm/agents/__init__.py
new file mode 100644
index 000000000..1a02765cd
--- /dev/null
+++ b/python/infinilm/agents/__init__.py
@@ -0,0 +1,30 @@
+"""
+Agent support for InfiniLM: tool calls and reasoning parsing.
+Supports GLM-4, Llama-3.1+, and Qwen3 tool call formats.
+"""
+
+from .function_call_parser import FunctionCallParser
+from .message_adapter import adapt_messages
+from .protocol import DeltaMessage, Function, Tool, ToolChoice
+from .reasoning_parser import ReasoningParser
+from .stream_parser import (
+ AgentDelta,
+ AgentStreamParser,
+ parse_full_response,
+)
+from .types import StreamingParseResult, ToolCallItem
+
+__all__ = [
+ "ToolCallItem",
+ "StreamingParseResult",
+ "Tool",
+ "ToolChoice",
+ "Function",
+ "DeltaMessage",
+ "FunctionCallParser",
+ "ReasoningParser",
+ "AgentDelta",
+ "AgentStreamParser",
+ "parse_full_response",
+ "adapt_messages",
+]
diff --git a/python/infinilm/agents/anthropic.py b/python/infinilm/agents/anthropic.py
new file mode 100644
index 000000000..24816158c
--- /dev/null
+++ b/python/infinilm/agents/anthropic.py
@@ -0,0 +1,538 @@
+"""
+Anthropic Messages API protocol support.
+
+Everything needed to speak the Anthropic protocol on top of the internal
+OpenAI-format pipeline: request/response models, request conversion, and the
+streaming SSE converter. Kept free of HTTP/engine concerns so it stays
+unit-testable in isolation.
+"""
+
+import json
+import uuid
+from typing import Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+
+# ---------- request models ----------
+
+
+class AnthropicTextBlock(BaseModel):
+ type: Literal["text"] = "text"
+ text: str
+
+
+class AnthropicToolUseBlock(BaseModel):
+ type: Literal["tool_use"] = "tool_use"
+ id: str
+ name: str
+ input: dict = Field(default_factory=dict)
+
+
+class AnthropicThinkingBlock(BaseModel):
+ type: Literal["thinking"] = "thinking"
+ thinking: str
+
+
+class AnthropicToolResultBlock(BaseModel):
+ type: Literal["tool_result"] = "tool_result"
+ tool_use_id: Optional[str] = None
+ content: Optional[Union[str, list["AnthropicContentBlock"]]] = None
+ is_error: Optional[bool] = None
+
+
+AnthropicContentBlock = Union[
+ AnthropicTextBlock,
+ AnthropicThinkingBlock,
+ AnthropicToolUseBlock,
+ AnthropicToolResultBlock,
+]
+
+
+class AnthropicMessage(BaseModel):
+ role: Literal["user", "assistant", "system"]
+ content: Union[str, list[AnthropicContentBlock]]
+
+
+class AnthropicMessagesRequest(BaseModel):
+ model: str
+ messages: list[AnthropicMessage]
+ max_tokens: int
+ metadata: Optional[dict] = None
+ stop_sequences: Optional[list[str]] = None
+ stream: Optional[bool] = False
+ system: Optional[Union[str, list[AnthropicTextBlock]]] = None
+ temperature: Optional[float] = None
+ tool_choice: Optional[dict] = None
+ tools: Optional[list[dict]] = None
+ top_k: Optional[int] = None
+ top_p: Optional[float] = None
+
+
+# ---------- SSE helpers ----------
+
+
+def anthropic_sse_event(event_type: str, data: dict) -> str:
+ """Wrap a dict as an Anthropic-style SSE event."""
+ return f"event: {event_type}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
+
+
+def anthropic_error_body(message: str) -> dict:
+ """Body of an Anthropic-style error response."""
+ return {
+ "type": "error",
+ "error": {"type": "invalid_request_error", "message": message},
+ }
+
+
+def parse_openai_sse_line(raw: str) -> Optional[dict]:
+ """Parse one OpenAI SSE line into its chunk dict; None if not a data chunk."""
+ if not raw.startswith("data: ") or raw.startswith("data: [DONE]"):
+ return None
+ try:
+ return json.loads(raw[6:].strip())
+ except (json.JSONDecodeError, ValueError):
+ return None
+
+
+# ---------- request / response conversion ----------
+
+
+def convert_anthropic_request(anthropic_req: AnthropicMessagesRequest) -> dict:
+ """Convert an Anthropic Messages request to an OpenAI chat completion dict."""
+ openai_messages = []
+
+ # --- System message ---
+ system_parts = []
+ if anthropic_req.system is not None:
+ if isinstance(anthropic_req.system, str):
+ if anthropic_req.system.strip():
+ system_parts.append(anthropic_req.system)
+ else:
+ for block in anthropic_req.system:
+ if block.type == "text" and block.text:
+ system_parts.append(block.text)
+ # Also pick up inline system messages
+ for msg in anthropic_req.messages:
+ if msg.role == "system":
+ if isinstance(msg.content, str) and msg.content.strip():
+ system_parts.append(msg.content)
+ else:
+ for block in msg.content or []:
+ if isinstance(block, AnthropicTextBlock) and block.text:
+ system_parts.append(block.text)
+ if system_parts:
+ openai_messages.append({"role": "system", "content": "\n".join(system_parts)})
+
+ # --- User / Assistant messages ---
+ for msg in anthropic_req.messages:
+ if msg.role == "system":
+ continue
+ if isinstance(msg.content, str):
+ openai_messages.append({"role": msg.role, "content": msg.content})
+ continue
+
+ openai_msg: dict = {"role": msg.role}
+ content_parts: list[dict] = []
+ tool_calls: list[dict] = []
+
+ for block in msg.content:
+ if block.type == "text" and block.text is not None:
+ content_parts.append({"type": "text", "text": block.text})
+ elif block.type == "image":
+ # Best-effort image passthrough
+ content_parts.append(block.model_dump(exclude_none=True))
+ elif block.type == "thinking":
+ # Reasoning content is model-internal; drop it for OpenAI backend.
+ pass
+ elif block.type == "tool_use":
+ tool_calls.append(
+ {
+ "id": block.id or f"call_{uuid.uuid4().hex}",
+ "type": "function",
+ "function": {
+ "name": block.name or "",
+ "arguments": json.dumps(block.input or {}),
+ },
+ }
+ )
+ elif block.type == "tool_result":
+ tool_content = block.content
+ tool_text = ""
+ if isinstance(tool_content, str):
+ tool_text = tool_content
+ elif isinstance(tool_content, list):
+ texts = [
+ b.text
+ for b in tool_content
+ if isinstance(b, AnthropicTextBlock)
+ ]
+ tool_text = "\n".join(texts)
+
+ tool_call_id = block.tool_use_id or ""
+ # Flush any pending user content first
+ if content_parts and msg.role == "user":
+ if len(content_parts) == 1 and content_parts[0]["type"] == "text":
+ openai_messages.append(
+ {"role": "user", "content": content_parts[0]["text"]}
+ )
+ else:
+ openai_messages.append(
+ {"role": "user", "content": list(content_parts)}
+ )
+ content_parts.clear()
+
+ if msg.role == "user":
+ openai_messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": tool_text,
+ }
+ )
+ else:
+ content_parts.append(
+ {"type": "text", "text": f"Tool result: {tool_text}"}
+ )
+
+ if tool_calls:
+ openai_msg["tool_calls"] = tool_calls
+ if content_parts:
+ if len(content_parts) == 1 and content_parts[0]["type"] == "text":
+ openai_msg["content"] = content_parts[0]["text"]
+ else:
+ openai_msg["content"] = content_parts
+ elif tool_calls:
+ pass # assistant message with only tool_calls
+ elif msg.role == "user":
+ continue # already emitted as tool messages
+ else:
+ openai_msg["content"] = "" # empty assistant placeholder
+
+ openai_messages.append(openai_msg)
+
+ data: dict = {
+ "messages": openai_messages,
+ "model": anthropic_req.model,
+ "max_tokens": anthropic_req.max_tokens,
+ "stream": anthropic_req.stream or False,
+ }
+ if anthropic_req.temperature is not None:
+ data["temperature"] = anthropic_req.temperature
+ if anthropic_req.top_p is not None:
+ data["top_p"] = anthropic_req.top_p
+ if anthropic_req.top_k is not None:
+ data["top_k"] = anthropic_req.top_k
+ if anthropic_req.stop_sequences is not None:
+ data["stop"] = anthropic_req.stop_sequences
+
+ # Tools
+ if anthropic_req.tools:
+ openai_tools = []
+ for tool in anthropic_req.tools:
+ openai_tools.append(
+ {
+ "type": "function",
+ "function": {
+ "name": tool.get("name", ""),
+ "description": tool.get("description", ""),
+ "parameters": tool.get("input_schema", {}),
+ },
+ }
+ )
+ data["tools"] = openai_tools
+ tc = anthropic_req.tool_choice
+ if tc is None:
+ data["tool_choice"] = "auto"
+ elif tc.get("type") == "none":
+ data["tool_choice"] = "none"
+ elif tc.get("type") == "any":
+ data["tool_choice"] = "required"
+ elif tc.get("type") == "tool":
+ data["tool_choice"] = {
+ "type": "function",
+ "function": {"name": tc.get("name", "")},
+ }
+ else:
+ data["tool_choice"] = "auto"
+
+ return data
+
+
+def convert_openai_to_anthropic_response(response: dict, model_id: str) -> dict:
+ """Convert an OpenAI chat completion response to an Anthropic Messages response."""
+ choices = response.get("choices", [])
+ if not choices:
+ return {
+ "id": f"msg_{uuid.uuid4().hex}",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": ""}],
+ "model": model_id,
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 0, "output_tokens": 0},
+ }
+
+ choice = choices[0]
+ message = choice.get("message", {})
+ content: list[dict] = []
+
+ # Reasoning content -> thinking block (best-effort)
+ reasoning = message.get("reasoning_content")
+ if reasoning:
+ content.append({"type": "thinking", "thinking": reasoning})
+
+ # Text content
+ text = message.get("content", "")
+ if text:
+ content.append({"type": "text", "text": text})
+
+ # Tool calls -> tool_use blocks
+ for tc in message.get("tool_calls", []):
+ raw_args = tc.get("function", {}).get("arguments", "")
+ try:
+ tool_input = json.loads(raw_args) if raw_args else {}
+ except (json.JSONDecodeError, TypeError):
+ tool_input = {}
+ content.append(
+ {
+ "type": "tool_use",
+ "id": tc.get("id", f"call_{uuid.uuid4().hex}"),
+ "name": tc.get("function", {}).get("name", ""),
+ "input": tool_input,
+ }
+ )
+
+ if not content:
+ content.append({"type": "text", "text": ""})
+
+ finish_reason = choice.get("finish_reason") or "stop"
+ stop_reason_map = {
+ "stop": "end_turn",
+ "length": "max_tokens",
+ "tool_calls": "tool_use",
+ }
+ stop_reason = stop_reason_map.get(finish_reason, "end_turn")
+
+ usage = response.get("usage", {})
+ return {
+ "id": f"msg_{uuid.uuid4().hex}",
+ "type": "message",
+ "role": "assistant",
+ "content": content,
+ "model": model_id,
+ "stop_reason": stop_reason,
+ "usage": {
+ "input_tokens": usage.get("prompt_tokens", 0),
+ "output_tokens": usage.get("completion_tokens", 0),
+ },
+ }
+
+
+# ---------- streaming conversion ----------
+
+
+class AnthropicStreamConverter:
+ """Convert OpenAI chat-completion stream chunks into Anthropic SSE events.
+
+ Content blocks are tracked with an explicit active block type and a
+ monotonically increasing block index: whenever the kind of output changes
+ (thinking -> text -> tool_use, or a new tool call starts), the active
+ block is closed and a new one is opened at the next index, so the emitted
+ ``content_block_start/delta/stop`` sequence is always valid.
+ """
+
+ STOP_REASON_MAP = {
+ "stop": "end_turn",
+ "length": "max_tokens",
+ "tool_calls": "tool_use",
+ }
+
+ def __init__(self, message_id: str, model: str):
+ self._message_id = message_id
+ self._model = model
+ # Type of the currently open content block, if any:
+ # None | "thinking" | "text" | "tool_use"
+ self._active_block: Optional[str] = None
+ # Index of the currently open block; increases monotonically.
+ self._block_index = -1
+ # OpenAI tool_calls index of the currently open tool_use block.
+ self._open_tool_idx: Optional[int] = None
+ self._finish_reason: Optional[str] = None
+ self._usage: Optional[dict] = None
+
+ def begin(self) -> list:
+ """Events to emit before the first chunk (message_start)."""
+ return [
+ anthropic_sse_event(
+ "message_start",
+ {
+ "type": "message_start",
+ "message": {
+ "id": self._message_id,
+ "type": "message",
+ "role": "assistant",
+ "content": [],
+ "model": self._model,
+ "usage": {"input_tokens": 0, "output_tokens": 0},
+ },
+ },
+ )
+ ]
+
+ def feed(self, chunk: dict) -> list:
+ """Events for one OpenAI-format stream chunk."""
+ events = []
+ choices = chunk.get("choices") or []
+ if not choices:
+ return events
+
+ delta = choices[0].get("delta") or {}
+ self._finish_reason = choices[0].get("finish_reason") or self._finish_reason
+ if chunk.get("usage"):
+ self._usage = chunk["usage"]
+
+ # -- Reasoning content (thinking block) --
+ reasoning = delta.get("reasoning_content")
+ if reasoning:
+ if self._active_block != "thinking":
+ events.extend(
+ self._switch_block("thinking", {"type": "thinking", "thinking": ""})
+ )
+ events.append(
+ anthropic_sse_event(
+ "content_block_delta",
+ {
+ "type": "content_block_delta",
+ "index": self._block_index,
+ "delta": {"type": "thinking_delta", "thinking": reasoning},
+ },
+ )
+ )
+
+ # -- Text content --
+ text = delta.get("content")
+ if text:
+ if self._active_block != "text":
+ events.extend(self._switch_block("text", {"type": "text", "text": ""}))
+ events.append(
+ anthropic_sse_event(
+ "content_block_delta",
+ {
+ "type": "content_block_delta",
+ "index": self._block_index,
+ "delta": {"type": "text_delta", "text": text},
+ },
+ )
+ )
+
+ # -- Tool calls --
+ for tc in delta.get("tool_calls") or []:
+ idx = tc.get("index", 0)
+ if self._active_block != "tool_use" or self._open_tool_idx != idx:
+ events.extend(
+ self._switch_block(
+ "tool_use",
+ {
+ "type": "tool_use",
+ "id": tc.get("id") or f"call_{idx}",
+ "name": tc.get("function", {}).get("name", ""),
+ "input": {},
+ },
+ )
+ )
+ self._open_tool_idx = idx
+
+ args = tc.get("function", {}).get("arguments", "")
+ if args:
+ events.append(
+ anthropic_sse_event(
+ "content_block_delta",
+ {
+ "type": "content_block_delta",
+ "index": self._block_index,
+ "delta": {
+ "type": "input_json_delta",
+ "partial_json": args,
+ },
+ },
+ )
+ )
+
+ return events
+
+ def end(self) -> list:
+ """Closing events after the chunk stream ends."""
+ events = []
+ if self._active_block is not None:
+ events.append(
+ anthropic_sse_event(
+ "content_block_stop",
+ {"type": "content_block_stop", "index": self._block_index},
+ )
+ )
+ stop_reason = self.STOP_REASON_MAP.get(
+ self._finish_reason or "stop", "end_turn"
+ )
+ usage = self._usage or {}
+ events.append(
+ anthropic_sse_event(
+ "message_delta",
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": stop_reason},
+ "usage": {
+ "input_tokens": usage.get("prompt_tokens", 0),
+ "output_tokens": usage.get("completion_tokens", 0),
+ },
+ },
+ )
+ )
+ events.append(anthropic_sse_event("message_stop", {"type": "message_stop"}))
+ return events
+
+ def _switch_block(self, block_type: str, content_block: dict) -> list:
+ """Close the active block (if any) and open a new one."""
+ events = []
+ if self._active_block is not None:
+ events.append(
+ anthropic_sse_event(
+ "content_block_stop",
+ {"type": "content_block_stop", "index": self._block_index},
+ )
+ )
+ self._block_index += 1
+ self._active_block = block_type
+ if block_type != "tool_use":
+ self._open_tool_idx = None
+ events.append(
+ anthropic_sse_event(
+ "content_block_start",
+ {
+ "type": "content_block_start",
+ "index": self._block_index,
+ "content_block": content_block,
+ },
+ )
+ )
+ return events
+
+
+async def convert_openai_sse_stream(openai_stream, message_id: str, model: str):
+ """Convert an OpenAI SSE line stream into Anthropic SSE events.
+
+ ``openai_stream`` yields the raw ``data: ...`` lines produced by the
+ OpenAI-format chat stream; this generator yields Anthropic-format SSE
+ event strings.
+ """
+ converter = AnthropicStreamConverter(message_id=message_id, model=model)
+ for event in converter.begin():
+ yield event
+ async for raw in openai_stream:
+ if raw.startswith("data: [DONE]"):
+ break
+ chunk = parse_openai_sse_line(raw)
+ if chunk is not None:
+ for event in converter.feed(chunk):
+ yield event
+ for event in converter.end():
+ yield event
diff --git a/python/infinilm/agents/base_detector.py b/python/infinilm/agents/base_detector.py
new file mode 100644
index 000000000..86f82301b
--- /dev/null
+++ b/python/infinilm/agents/base_detector.py
@@ -0,0 +1,335 @@
+"""
+Base format detector for tool-call parsing.
+Removes structural_tag / XGrammar-dependent methods.
+"""
+
+import json
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Literal, Union
+
+from .protocol import Tool, ToolChoice
+from .types import StreamingParseResult, ToolCallItem
+from .utils import _find_common_prefix, _is_complete_json, _partial_json_loads
+
+logger = logging.getLogger(__name__)
+
+
+try:
+ from partial_json_parser.core.options import Allow
+
+ PARTIAL_JSON_AVAILABLE = True
+except ImportError:
+ Allow = None
+ PARTIAL_JSON_AVAILABLE = False
+
+
+class BaseFormatDetector(ABC):
+ """Base class providing two sets of interfaces: one-time and streaming incremental."""
+
+ def __init__(self):
+ # Streaming state management
+ self._buffer = ""
+ # Stores complete tool call info for each tool being parsed.
+ self.prev_tool_call_arr: List[Dict] = []
+ # Index of currently streaming tool call.
+ self.current_tool_id: int = -1
+ # Flag for whether current tool's name has been sent to client.
+ self.current_tool_name_sent: bool = False
+ # Tracks raw JSON string content streamed to client for each tool's arguments.
+ self.streamed_args_for_tool: List[str] = []
+
+ # Token configuration (override in subclasses)
+ self.bot_token = ""
+ self.eot_token = ""
+ self.tool_call_separator = ", "
+
+ def clear(self):
+ self._buffer = ""
+ self.prev_tool_call_arr = []
+ self.current_tool_id = -1
+ self.current_tool_name_sent = False
+ self.streamed_args_for_tool = []
+ if hasattr(self, "_tool_indices"):
+ delattr(self, "_tool_indices")
+
+ def _get_tool_indices(self, tools: List[Tool]) -> Dict[str, int]:
+ """Get a mapping of tool names to their indices in the tools list."""
+ return {
+ tool.function.name: i for i, tool in enumerate(tools) if tool.function.name
+ }
+
+ def parse_base_json(
+ self, action: Any, tools: List[Tool], base_index: int = -1
+ ) -> List[ToolCallItem]:
+ tool_indices = self._get_tool_indices(tools)
+ if not isinstance(action, list):
+ action = [action]
+
+ results = []
+ for i, act in enumerate(action):
+ name = act.get("name")
+ if not (name and name in tool_indices):
+ logger.warning(f"Model attempted to call undefined function: {name}")
+ continue
+
+ if base_index >= 0:
+ tool_index = base_index + i
+ else:
+ tool_index = tool_indices.get(name, -1)
+
+ results.append(
+ ToolCallItem(
+ tool_index=tool_index,
+ name=name,
+ parameters=json.dumps(
+ act.get("parameters") or act.get("arguments", {}),
+ ensure_ascii=False,
+ ),
+ )
+ )
+
+ return results
+
+ @abstractmethod
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ """Parses the text in one shot."""
+ action = json.loads(text)
+ return StreamingParseResult(calls=self.parse_base_json(action, tools))
+
+ def _ends_with_partial_token(self, buffer: str, bot_token: str) -> int:
+ """Check if buffer ends with a partial bot_token."""
+ for i in range(1, min(len(buffer) + 1, len(bot_token))):
+ if bot_token.startswith(buffer[-i:]):
+ return i
+ return 0
+
+ def parse_streaming_increment(
+ self, new_text: str, tools: List[Tool]
+ ) -> StreamingParseResult:
+ """
+ Streaming incremental parsing with tool validation.
+ Base implementation works best with formats where:
+ 1. bot_token is followed immediately by JSON
+ 2. JSON can be parsed incrementally using partial_json_loads
+ 3. Multiple tool calls are separated by "; " or ", "
+ """
+ self._buffer += new_text
+ current_text = self._buffer
+
+ if not (
+ self.has_tool_call(current_text)
+ or (
+ self.current_tool_id > 0
+ and current_text.startswith(self.tool_call_separator)
+ )
+ ):
+ if not self._ends_with_partial_token(self._buffer, self.bot_token):
+ normal_text = self._buffer
+ self._buffer = ""
+ if self.eot_token in normal_text:
+ normal_text = normal_text.replace(self.eot_token, "")
+ return StreamingParseResult(normal_text=normal_text)
+ else:
+ return StreamingParseResult()
+
+ if not hasattr(self, "_tool_indices"):
+ self._tool_indices = self._get_tool_indices(tools)
+
+ flags = (
+ Allow.ALL
+ if (Allow is not None and self.current_tool_name_sent)
+ else (Allow.ALL & ~Allow.STR if Allow is not None else None)
+ )
+ if flags is None:
+ flags = 0 # fallback when partial_json_parser unavailable
+
+ try:
+ try:
+ used_separator_branch = False
+ if self.current_tool_id > 0 and current_text.startswith(
+ self.tool_call_separator
+ ):
+ start_idx = len(self.tool_call_separator)
+ used_separator_branch = True
+ else:
+ tool_call_pos = current_text.find(self.bot_token)
+ if tool_call_pos != -1:
+ start_idx = tool_call_pos + len(self.bot_token)
+ else:
+ start_idx = 0
+
+ if start_idx >= len(current_text):
+ return StreamingParseResult()
+
+ try:
+ obj, end_idx = _partial_json_loads(current_text[start_idx:], flags)
+ except (Exception, json.JSONDecodeError):
+ if used_separator_branch and self.bot_token in current_text:
+ start_idx = current_text.find(self.bot_token) + len(
+ self.bot_token
+ )
+ if start_idx >= len(current_text):
+ return StreamingParseResult()
+ obj, end_idx = _partial_json_loads(
+ current_text[start_idx:], flags
+ )
+ else:
+ raise
+
+ is_current_complete = _is_complete_json(
+ current_text[start_idx : start_idx + end_idx]
+ )
+
+ # A complete JSON object without a "name" field can never
+ # become a tool call; release the buffer as normal text
+ # instead of holding it back forever.
+ if is_current_complete and "name" not in obj:
+ normal_text = self._buffer
+ self._buffer = ""
+ if self.eot_token in normal_text:
+ normal_text = normal_text.replace(self.eot_token, "")
+ return StreamingParseResult(normal_text=normal_text)
+
+ if "name" in obj and obj["name"] not in self._tool_indices:
+ self._buffer = ""
+ self.current_tool_id = -1
+ self.current_tool_name_sent = False
+ if self.streamed_args_for_tool:
+ self.streamed_args_for_tool.pop()
+ return StreamingParseResult()
+
+ if "parameters" in obj:
+ obj["arguments"] = obj["parameters"]
+
+ current_tool_call = obj
+
+ except (Exception, json.JSONDecodeError):
+ return StreamingParseResult()
+
+ if not current_tool_call:
+ return StreamingParseResult()
+
+ if not self.current_tool_name_sent:
+ function_name = current_tool_call.get("name")
+
+ if function_name and function_name in self._tool_indices:
+ if self.current_tool_id == -1:
+ self.current_tool_id = 0
+ self.streamed_args_for_tool.append("")
+ elif self.current_tool_id >= len(self.streamed_args_for_tool):
+ while len(self.streamed_args_for_tool) <= self.current_tool_id:
+ self.streamed_args_for_tool.append("")
+
+ res = StreamingParseResult(
+ calls=[
+ ToolCallItem(
+ tool_index=self.current_tool_id,
+ name=function_name,
+ parameters="",
+ )
+ ],
+ )
+ self.current_tool_name_sent = True
+ else:
+ res = StreamingParseResult()
+
+ else:
+ cur_arguments = current_tool_call.get("arguments")
+ res = StreamingParseResult()
+
+ if cur_arguments is not None:
+ sent = len(self.streamed_args_for_tool[self.current_tool_id])
+ cur_args_json = json.dumps(cur_arguments, ensure_ascii=False)
+ prev_arguments = None
+ if self.current_tool_id < len(self.prev_tool_call_arr):
+ prev_arguments = self.prev_tool_call_arr[
+ self.current_tool_id
+ ].get("arguments")
+
+ argument_diff = None
+ completing_tool_id = self.current_tool_id
+
+ if is_current_complete:
+ argument_diff = cur_args_json[sent:]
+ completing_tool_id = self.current_tool_id
+ self._buffer = current_text[start_idx + end_idx :]
+
+ elif prev_arguments:
+ prev_args_json = json.dumps(prev_arguments, ensure_ascii=False)
+ if cur_args_json != prev_args_json:
+ prefix = _find_common_prefix(prev_args_json, cur_args_json)
+ argument_diff = prefix[sent:]
+
+ if self.current_tool_id >= 0:
+ while len(self.prev_tool_call_arr) <= self.current_tool_id:
+ self.prev_tool_call_arr.append({})
+ self.prev_tool_call_arr[self.current_tool_id] = (
+ current_tool_call
+ )
+
+ if is_current_complete:
+ self.current_tool_name_sent = False
+ self.current_tool_id += 1
+
+ if argument_diff is not None:
+ tool_index_to_use = (
+ completing_tool_id
+ if is_current_complete
+ else self.current_tool_id
+ )
+ res = StreamingParseResult(
+ calls=[
+ ToolCallItem(
+ tool_index=tool_index_to_use,
+ parameters=argument_diff,
+ )
+ ],
+ )
+ self.streamed_args_for_tool[tool_index_to_use] += argument_diff
+
+ return res
+
+ except Exception as e:
+ logger.error(f"Error in parse_streaming_increment: {e}")
+ return StreamingParseResult()
+
+ @abstractmethod
+ def has_tool_call(self, text: str) -> bool:
+ """Check if the given text contains function call markers."""
+ raise NotImplementedError()
+
+ def finish(self, tools: List[Tool]) -> StreamingParseResult:
+ """Called once when the stream ends; flush any buffered state."""
+ if self._buffer:
+ return self.parse_streaming_increment("", tools)
+ return StreamingParseResult()
+
+ def supports_structural_tag(self) -> bool:
+ return False
+
+ def parses_required_natively(self) -> bool:
+ return False
+
+ @abstractmethod
+ def structure_info(self):
+ """Return a function that creates StructureInfo for constrained generation."""
+ raise NotImplementedError()
+
+ def get_structural_tag(
+ self,
+ tools: Union[List[Tool], None] = None,
+ tool_choice: Union[ToolChoice, Literal["auto", "required"]] = "auto",
+ thinking_mode: bool = False,
+ parallel_tool_calls: bool = True,
+ ):
+ """Return a model-native structural tag when supported. (Disabled in InfiniLM MVP)"""
+ return None
+
+ def get_auto_tool_call_structural_tag(
+ self,
+ tools: Union[List[Tool], None] = None,
+ thinking_mode: bool = False,
+ parallel_tool_calls: bool = True,
+ ):
+ return None
diff --git a/python/infinilm/agents/detectors/__init__.py b/python/infinilm/agents/detectors/__init__.py
new file mode 100644
index 000000000..edbcf2da2
--- /dev/null
+++ b/python/infinilm/agents/detectors/__init__.py
@@ -0,0 +1,13 @@
+"""Tool call detectors for specific model families."""
+
+from .glm4_chat_0414_detector import Glm4Chat0414Detector
+from .glm4_moe_detector import Glm4MoeDetector
+from .llama32_detector import Llama32Detector
+from .qwen3_xml_detector import Qwen3XmlDetector
+
+__all__ = [
+ "Glm4Chat0414Detector",
+ "Glm4MoeDetector",
+ "Llama32Detector",
+ "Qwen3XmlDetector",
+]
diff --git a/python/infinilm/agents/detectors/glm4_chat_0414_detector.py b/python/infinilm/agents/detectors/glm4_chat_0414_detector.py
new file mode 100644
index 000000000..0c9be9d70
--- /dev/null
+++ b/python/infinilm/agents/detectors/glm4_chat_0414_detector.py
@@ -0,0 +1,304 @@
+"""
+Detector for GLM-4-9B-Chat-0414 style models ("metadata" tool call format).
+
+The official protocol (see the THUDM/glm-4-9b-chat-0414 model card):
+
+- tools are listed in a ``# 可用工具`` system section of the prompt;
+- the model calls a tool by emitting the function name on one line and a
+ JSON object with the arguments on the next line::
+
+ get_weather
+ {"city": "北京"}
+
+- the official decoder detects calls with the regex
+ ``([^\\n`]*?)\\n({.*?})(?=\\w*\\n|$)`` and parses the arguments with
+ ``json.loads`` (falling back to ``ast.literal_eval``);
+- tool results are fed back with the ``observation`` role, and parallel
+ calls are separated by ``<|assistant|>`` markers inside the completion.
+"""
+
+import json
+import logging
+import re
+from typing import Dict, List, Optional, Tuple
+
+from infinilm.agents.base_detector import BaseFormatDetector
+from infinilm.agents.protocol import Tool
+from infinilm.agents.types import StreamingParseResult, _GetInfoFunc
+from infinilm.agents.utils import safe_literal_eval
+
+logger = logging.getLogger(__name__)
+
+
+# Official detection pattern from the model card (used by has_tool_call()).
+_OFFICIAL_FC_PATTERN = re.compile(r"([^\n`]*?)\n({.*?})(?=\w*\n|$)", re.DOTALL)
+
+
+class Glm4Chat0414Detector(BaseFormatDetector):
+ """Detector for the GLM-4-9B-0414 metadata-style tool call format.
+
+ Format structure::
+
+ function_name
+ {"arg": "value"}
+
+ with an optional ``<|assistant|>`` marker between parallel calls.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.bot_token = ""
+ self.eot_token = "<|assistant|>"
+ self.tool_call_separator = "\n"
+
+ # ------------------------------------------------------------------
+ # helpers
+ # ------------------------------------------------------------------
+
+ @staticmethod
+ def _scan_json_object(text: str, start: int) -> int:
+ """Return the end offset of the JSON object starting at ``start``.
+
+ Returns -1 while the object is still incomplete.
+ """
+ depth = 0
+ in_string = False
+ escaped = False
+ for i in range(start, len(text)):
+ ch = text[i]
+ if in_string:
+ if escaped:
+ escaped = False
+ elif ch == "\\":
+ escaped = True
+ elif ch == '"':
+ in_string = False
+ continue
+ if ch == '"':
+ in_string = True
+ elif ch == "{":
+ depth += 1
+ elif ch == "}":
+ depth -= 1
+ if depth == 0:
+ return i + 1
+ return -1
+
+ @staticmethod
+ def _parse_arguments(args_str: str):
+ """Parse tool arguments like the official example does."""
+ try:
+ return json.loads(args_str)
+ except json.JSONDecodeError:
+ pass
+ try:
+ return safe_literal_eval(args_str)
+ except (ValueError, SyntaxError):
+ return None
+
+ @staticmethod
+ def _could_be_tool_name(line: str, tool_indices: Dict[str, int]) -> bool:
+ """Whether ``line`` can still (become) a known tool name."""
+ line = line.strip()
+ if not line or "`" in line:
+ return False
+ return any(name == line or name.startswith(line) for name in tool_indices)
+
+ def _find_candidate(
+ self, text: str, pos: int, tool_indices: Dict[str, int]
+ ) -> Tuple[int, int, Optional[str], bool]:
+ """Locate the next ``name\\n{`` candidate at or after ``pos``.
+
+ Returns ``(brace_pos, name_line_start, name, True)`` when a candidate
+ exists, otherwise ``(hold_from, -1, None, False)`` where ``hold_from``
+ marks how much of ``text[pos:]`` is safe to emit as normal text (the
+ rest is held back because it could still grow into a tool call or a
+ ``<|assistant|>`` call separator).
+ """
+ search_from = pos
+ while True:
+ brace_idx = text.find("{", search_from)
+ if brace_idx == -1 or brace_idx == 0:
+ break
+ if text[brace_idx - 1] != "\n":
+ search_from = brace_idx + 1
+ continue
+ name_line_start = max(text.rfind("\n", pos, brace_idx - 1), pos - 1) + 1
+ name = text[name_line_start : brace_idx - 1]
+ if "`" in name or not name.strip():
+ search_from = brace_idx + 1
+ continue
+ return brace_idx, name_line_start, name.strip(), True
+
+ # No candidate. Hold back a tail that could still grow into
+ # "name\n{" with future increments: the last line, but only if it
+ # can still become a known tool name.
+ hold_from = len(text)
+ if text.endswith("\n"):
+ prev_nl = text.rfind("\n", pos, len(text) - 1)
+ line_before = text[prev_nl + 1 : len(text) - 1]
+ if self._could_be_tool_name(line_before, tool_indices):
+ hold_from = prev_nl + 1
+ else:
+ last_nl = text.rfind("\n", pos)
+ tail = text[last_nl + 1 :]
+ if self._could_be_tool_name(tail, tool_indices):
+ hold_from = last_nl + 1
+
+ # The trailing text could also be a partial "<|assistant|>"
+ # separator; never release characters that are a suffix of it.
+ for i in range(min(len(self.eot_token), len(text) - pos), 0, -1):
+ if text.endswith(self.eot_token[:i]):
+ hold_from = min(hold_from, len(text) - i)
+ break
+ return hold_from, -1, None, False
+
+ # ------------------------------------------------------------------
+ # one-shot parsing
+ # ------------------------------------------------------------------
+
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ tool_indices = self._get_tool_indices(tools)
+ text = text.replace(self.eot_token, "\n")
+
+ normal_parts: List[str] = []
+ calls = []
+ pos = 0
+ while pos < len(text):
+ nl_brace = text.find("\n{", pos)
+ if nl_brace == -1:
+ normal_parts.append(text[pos:])
+ break
+ name_line_start = max(text.rfind("\n", pos, nl_brace), pos - 1) + 1
+ name = text[name_line_start:nl_brace].strip()
+ json_end = self._scan_json_object(text, nl_brace + 1)
+ if json_end == -1:
+ # Truncated JSON at end of text: keep everything verbatim.
+ normal_parts.append(text[pos:])
+ break
+
+ if name in tool_indices:
+ arguments = self._parse_arguments(text[nl_brace + 1 : json_end])
+ if arguments is not None:
+ normal_parts.append(text[pos:name_line_start])
+ calls.extend(
+ self.parse_base_json(
+ {"name": name, "parameters": arguments},
+ tools,
+ len(calls),
+ )
+ )
+ else:
+ # Unparseable arguments: keep the block as normal text.
+ normal_parts.append(text[pos:json_end])
+ else:
+ # Not a known tool: keep the text verbatim.
+ normal_parts.append(text[pos:json_end])
+ pos = json_end
+
+ normal_text = "".join(normal_parts).strip()
+ return StreamingParseResult(normal_text=normal_text, calls=calls)
+
+ # ------------------------------------------------------------------
+ # streaming parsing
+ # ------------------------------------------------------------------
+
+ def parse_streaming_increment(
+ self, new_text: str, tools: List[Tool]
+ ) -> StreamingParseResult:
+ self._buffer += new_text
+ tool_indices = self._get_tool_indices(tools)
+
+ # Call separators never belong to user-visible text. A separator may
+ # arrive split across increments: complete ones are replaced right
+ # away, while a trailing partial separator is held back (safe_end)
+ # until the next increment decides what it is.
+ if self.eot_token in self._buffer:
+ self._buffer = self._buffer.replace(self.eot_token, "\n")
+ safe_end = len(self._buffer)
+ for k in range(min(len(self.eot_token) - 1, len(self._buffer)), 0, -1):
+ if self._buffer.endswith(self.eot_token[:k]):
+ safe_end = len(self._buffer) - k
+ break
+
+ calls = []
+ normal_chunks: List[str] = []
+ pos = 0
+ while True:
+ brace_pos, name_line_start, name, found = self._find_candidate(
+ self._buffer, pos, tool_indices
+ )
+ if not found:
+ emit_end = min(brace_pos, safe_end)
+ if emit_end > pos:
+ normal_chunks.append(self._buffer[pos:emit_end])
+ self._buffer = self._buffer[emit_end:]
+ return StreamingParseResult(
+ normal_text="".join(normal_chunks), calls=calls
+ )
+
+ json_end = self._scan_json_object(self._buffer, brace_pos)
+
+ if name in tool_indices:
+ if json_end == -1:
+ # Incomplete JSON of a known tool: hold it back until the
+ # object is complete (or the stream ends).
+ if name_line_start > pos:
+ normal_chunks.append(self._buffer[pos:name_line_start])
+ self._buffer = self._buffer[name_line_start:]
+ return StreamingParseResult(
+ normal_text="".join(normal_chunks), calls=calls
+ )
+ arguments = self._parse_arguments(self._buffer[brace_pos:json_end])
+ if arguments is not None:
+ if name_line_start > pos:
+ normal_chunks.append(self._buffer[pos:name_line_start])
+ calls.extend(
+ self.parse_base_json(
+ {"name": name, "parameters": arguments},
+ tools,
+ len(calls),
+ )
+ )
+ else:
+ normal_chunks.append(self._buffer[pos:json_end])
+ pos = json_end
+ else:
+ # Unknown name: not a tool call; release it as normal text.
+ if json_end == -1:
+ emit_end = max(pos, min(safe_end, len(self._buffer)))
+ if emit_end > pos:
+ normal_chunks.append(self._buffer[pos:emit_end])
+ self._buffer = self._buffer[emit_end:]
+ return StreamingParseResult(
+ normal_text="".join(normal_chunks), calls=calls
+ )
+ normal_chunks.append(self._buffer[pos:json_end])
+ pos = json_end
+
+ def finish(self, tools: List[Tool]) -> StreamingParseResult:
+ """Flush remaining buffer at stream end."""
+ result = StreamingParseResult()
+ if self._buffer:
+ sp_result = self.parse_streaming_increment("", tools)
+ result.normal_text += sp_result.normal_text
+ result.calls.extend(sp_result.calls)
+ # Anything still buffered now is a truncated tool call; release
+ # it as text so no generated content is silently lost.
+ if self._buffer:
+ result.normal_text += self._buffer
+ self._buffer = ""
+ return result
+
+ # ------------------------------------------------------------------
+ # hooks
+ # ------------------------------------------------------------------
+
+ def has_tool_call(self, text: str) -> bool:
+ return bool(_OFFICIAL_FC_PATTERN.search(text))
+
+ def supports_structural_tag(self) -> bool:
+ return False
+
+ def structure_info(self) -> _GetInfoFunc:
+ raise NotImplementedError()
diff --git a/python/infinilm/agents/detectors/glm4_moe_detector.py b/python/infinilm/agents/detectors/glm4_moe_detector.py
new file mode 100644
index 000000000..5be8a98f2
--- /dev/null
+++ b/python/infinilm/agents/detectors/glm4_moe_detector.py
@@ -0,0 +1,415 @@
+"""
+Detector for GLM-4 and GLM-4.5 models.
+"""
+
+import json
+import logging
+import re
+from enum import Enum
+from typing import Any, Dict, List, Optional, Tuple
+
+from infinilm.agents.base_detector import BaseFormatDetector
+from infinilm.agents.protocol import Tool
+from infinilm.agents.types import StreamingParseResult, ToolCallItem, _GetInfoFunc
+from infinilm.agents.utils import (
+ _convert_to_number,
+ get_schema_properties,
+ infer_type_from_json_schema,
+ parse_arguments,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class StreamState(str, Enum):
+ INIT = "INIT"
+ BETWEEN = "BETWEEN"
+ IN_KEY = "IN_KEY"
+ WAITING_VALUE = "WAITING_VALUE"
+ IN_VALUE = "IN_VALUE"
+
+
+def get_argument_type_glm(
+ func_name: str, arg_key: str, defined_tools: List[Tool]
+) -> Optional[str]:
+ """Get the expected type of a function argument from tool definitions."""
+ name2tool = {tool.function.name: tool for tool in defined_tools}
+ if func_name not in name2tool:
+ return None
+ tool = name2tool[func_name]
+ properties = get_schema_properties(tool.function.parameters)
+ if arg_key not in properties:
+ return None
+ return infer_type_from_json_schema(properties[arg_key])
+
+
+class Glm4MoeDetector(BaseFormatDetector):
+ """
+ Detector for GLM-4.5 and GLM-4 models.
+ Assumes function call format:
+ get_weather
+ city
+ 北京
+ date
+ 2024-06-27
+
+ """
+
+ _STREAMING_PARTIAL_PATTERN = re.compile(
+ r"(.*?)(?:\\n|\n)(.*?)(|$)", re.DOTALL
+ )
+
+ def __init__(self):
+ super().__init__()
+ self.bot_token = ""
+ self.eot_token = ""
+ self.func_call_regex = r".*?"
+ self.func_detail_regex = re.compile(
+ r"(.*?)(?:\\n|\n)(.*)", re.DOTALL
+ )
+ self.func_arg_regex = re.compile(
+ r"(.*?)(?:\\n|\s)*(.*?)",
+ re.DOTALL,
+ )
+ self._last_arguments = ""
+ self.current_tool_id = -1
+ self.current_tool_name_sent = False
+ self._streamed_raw_length = 0
+ self._reset_streaming_state()
+
+ def _reset_streaming_state(self) -> None:
+ self._stream_state = StreamState.INIT
+ self._current_key = ""
+ self._current_value = ""
+ self._xml_tag_buffer = ""
+ self._is_first_param = True
+ self._value_started = False
+ self._cached_value_type: Optional[str] = None
+
+ def clear(self):
+ super().clear()
+ self._last_arguments = ""
+ self._streamed_raw_length = 0
+ self._reset_streaming_state()
+
+ def has_tool_call(self, text: str) -> bool:
+ return self.bot_token in text
+
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ idx = text.find(self.bot_token)
+ normal_text = text[:idx].strip() if idx != -1 else text
+ if self.bot_token not in text:
+ return StreamingParseResult(normal_text=normal_text, calls=[])
+ match_result_list = re.findall(self.func_call_regex, text, re.DOTALL)
+ calls = []
+ try:
+ for match_result in match_result_list:
+ func_detail = self.func_detail_regex.search(match_result)
+ if func_detail is None:
+ continue
+ func_name = func_detail.group(1) if func_detail.group(1) else ""
+ func_args = func_detail.group(2) if func_detail.group(2) else ""
+ pairs = self.func_arg_regex.findall(func_args)
+ arguments = self._parse_argument_pairs(pairs, func_name, tools)
+ match_result_obj = {"name": func_name, "parameters": arguments}
+ calls.extend(self.parse_base_json(match_result_obj, tools))
+ return StreamingParseResult(normal_text=normal_text, calls=calls)
+ except Exception as e:
+ logger.error(f"Error in detect_and_parse: {e}", exc_info=True)
+ return StreamingParseResult(normal_text=text)
+
+ def _get_value_type(self, func_name: str, key: str, tools: List[Tool]) -> str:
+ arg_type = get_argument_type_glm(func_name, key, tools)
+ if arg_type:
+ return arg_type
+ value_content = self._current_value.strip() if self._current_value else ""
+ if not value_content:
+ return "string"
+ try:
+ parsed = json.loads(value_content)
+ if isinstance(parsed, dict):
+ return "object"
+ elif isinstance(parsed, list):
+ return "array"
+ elif isinstance(parsed, bool):
+ return "boolean"
+ elif isinstance(parsed, (int, float)):
+ return "number"
+ elif isinstance(parsed, str):
+ if parsed.isdigit() or (
+ parsed.startswith("-") and parsed[1:].isdigit()
+ ):
+ return "number"
+ return "string"
+ except json.JSONDecodeError:
+ first_char = value_content[0] if value_content else ""
+ if first_char.isdigit() or first_char in ["-", "."]:
+ return "number"
+ elif first_char in ["{", "["]:
+ return "object"
+ elif first_char in ['"', "'"]:
+ return "string"
+ return "string"
+
+ def _format_value_complete(self, value: str, value_type: str) -> str:
+ if value_type == "string":
+ return json.dumps(value, ensure_ascii=False)
+ elif value_type == "number":
+ try:
+ num = _convert_to_number(value.strip())
+ return str(num)
+ except (ValueError, AttributeError):
+ logger.warning(
+ f"Failed to parse '{value}' as number, treating as string"
+ )
+ return json.dumps(str(value), ensure_ascii=False)
+ else:
+ return value
+
+ def _process_xml_to_json_streaming(
+ self, raw_increment: str, func_name: str, tools: List[Tool]
+ ) -> str:
+ json_output = ""
+ for char in raw_increment:
+ self._xml_tag_buffer += char
+ if self._stream_state in [StreamState.INIT, StreamState.BETWEEN]:
+ if self._xml_tag_buffer.endswith(""):
+ self._stream_state = StreamState.IN_KEY
+ self._current_key = ""
+ self._xml_tag_buffer = ""
+ json_output += "{" if self._is_first_param else ", "
+ self._is_first_param = False
+ elif self._stream_state == StreamState.IN_KEY:
+ if self._xml_tag_buffer.endswith(""):
+ self._current_key = self._xml_tag_buffer[:-10].strip()
+ self._xml_tag_buffer = ""
+ self._stream_state = StreamState.WAITING_VALUE
+ json_output += (
+ json.dumps(self._current_key, ensure_ascii=False) + ": "
+ )
+ elif self._stream_state == StreamState.WAITING_VALUE:
+ if self._xml_tag_buffer.endswith(""):
+ self._stream_state = StreamState.IN_VALUE
+ self._current_value = ""
+ self._xml_tag_buffer = ""
+ self._value_started = False
+ self._cached_value_type = self._get_value_type(
+ func_name, self._current_key, tools
+ )
+ elif self._stream_state == StreamState.IN_VALUE:
+ if self._xml_tag_buffer.endswith(""):
+ final_value = self._xml_tag_buffer[:-12]
+ self._current_value += final_value
+ value_type = self._cached_value_type or "string"
+ if self._value_started:
+ if final_value:
+ if value_type == "string":
+ json_output += json.dumps(
+ final_value, ensure_ascii=False
+ )[1:-1]
+ else:
+ json_output += final_value
+ if value_type == "string":
+ json_output += '"'
+ else:
+ json_output += self._format_value_complete(
+ self._current_value, value_type
+ )
+ self._xml_tag_buffer = ""
+ self._stream_state = StreamState.BETWEEN
+ self._current_value = ""
+ self._value_started = False
+ self._cached_value_type = None
+ else:
+ closing_tag = ""
+ is_potential_closing = len(self._xml_tag_buffer) <= len(
+ closing_tag
+ ) and closing_tag.startswith(self._xml_tag_buffer)
+ if not is_potential_closing:
+ content = self._xml_tag_buffer
+ value_type = self._cached_value_type or "string"
+ if value_type == "string":
+ if not self._value_started:
+ json_output += '"'
+ self._value_started = True
+ if content:
+ json_output += json.dumps(content, ensure_ascii=False)[
+ 1:-1
+ ]
+ self._current_value += content
+ self._xml_tag_buffer = ""
+ elif value_type == "number":
+ if content:
+ if not self._value_started:
+ self._value_started = True
+ json_output += content
+ self._current_value += content
+ self._xml_tag_buffer = ""
+ else:
+ if content:
+ if not self._value_started:
+ self._value_started = True
+ json_output += content
+ self._current_value += content
+ self._xml_tag_buffer = ""
+ return json_output
+
+ def parse_streaming_increment(
+ self, new_text: str, tools: List[Tool]
+ ) -> StreamingParseResult:
+ self._buffer += new_text
+ current_text = self._buffer
+ has_tool_call = self.bot_token in current_text
+ if not has_tool_call:
+ is_potential_start = any(
+ self.bot_token.startswith(current_text[-i:])
+ for i in range(1, min(len(current_text), len(self.bot_token)) + 1)
+ )
+ if not is_potential_start:
+ output_text = current_text
+ self._buffer = ""
+ if self.eot_token in output_text:
+ output_text = output_text.replace(self.eot_token, "")
+ return StreamingParseResult(normal_text=output_text)
+ else:
+ return StreamingParseResult(normal_text="", calls=[])
+
+ if not hasattr(self, "_tool_indices"):
+ self._tool_indices = self._get_tool_indices(tools)
+
+ calls: list[ToolCallItem] = []
+ try:
+ partial_match = self._STREAMING_PARTIAL_PATTERN.search(current_text)
+ if partial_match:
+ func_name_raw = partial_match.group(1)
+ func_args_raw = partial_match.group(2)
+ is_tool_end = partial_match.group(3)
+ if func_name_raw is None or not func_name_raw.strip():
+ return StreamingParseResult(normal_text="", calls=[])
+ func_name = func_name_raw.strip()
+ func_args_raw = func_args_raw.strip() if func_args_raw else ""
+ if self.current_tool_id == -1:
+ self.current_tool_id = 0
+ self.prev_tool_call_arr = []
+ self.streamed_args_for_tool = [""]
+ self._streamed_raw_length = 0
+ self.current_tool_name_sent = False
+ self._reset_streaming_state()
+ while len(self.prev_tool_call_arr) <= self.current_tool_id:
+ self.prev_tool_call_arr.append({})
+ while len(self.streamed_args_for_tool) <= self.current_tool_id:
+ self.streamed_args_for_tool.append("")
+ if not self.current_tool_name_sent:
+ calls.append(
+ ToolCallItem(
+ tool_index=self.current_tool_id,
+ name=func_name,
+ parameters="",
+ )
+ )
+ self.current_tool_name_sent = True
+ self._streamed_raw_length = 0
+ self._reset_streaming_state()
+ self.prev_tool_call_arr[self.current_tool_id] = {
+ "name": func_name,
+ "arguments": {},
+ }
+ if self.current_tool_name_sent:
+ current_raw_length = len(func_args_raw)
+ if current_raw_length > self._streamed_raw_length:
+ raw_increment = func_args_raw[self._streamed_raw_length :]
+ json_increment = self._process_xml_to_json_streaming(
+ raw_increment, func_name, tools
+ )
+ self._streamed_raw_length = current_raw_length
+ if json_increment:
+ calls.append(
+ ToolCallItem(
+ tool_index=self.current_tool_id,
+ name=None,
+ parameters=json_increment,
+ )
+ )
+ self._last_arguments += json_increment
+ self.streamed_args_for_tool[self.current_tool_id] += (
+ json_increment
+ )
+ if is_tool_end == self.eot_token:
+ if self._is_first_param:
+ empty_object = "{}"
+ calls.append(
+ ToolCallItem(
+ tool_index=self.current_tool_id,
+ name=None,
+ parameters=empty_object,
+ )
+ )
+ self._last_arguments += empty_object
+ self.streamed_args_for_tool[self.current_tool_id] += (
+ empty_object
+ )
+ else:
+ closing_brace = "}"
+ calls.append(
+ ToolCallItem(
+ tool_index=self.current_tool_id,
+ name=None,
+ parameters=closing_brace,
+ )
+ )
+ self._last_arguments += closing_brace
+ self.streamed_args_for_tool[self.current_tool_id] += (
+ closing_brace
+ )
+ try:
+ pairs = self.func_arg_regex.findall(func_args_raw)
+ if pairs:
+ arguments = self._parse_argument_pairs(
+ pairs, func_name, tools
+ )
+ self.prev_tool_call_arr[self.current_tool_id][
+ "arguments"
+ ] = arguments
+ except Exception as e:
+ logger.debug(
+ f"Failed to parse arguments: {e}", exc_info=True
+ )
+ self._buffer = current_text[partial_match.end(3) :]
+ result = StreamingParseResult(normal_text="", calls=calls)
+ self.current_tool_id += 1
+ self._last_arguments = ""
+ self.current_tool_name_sent = False
+ self._streamed_raw_length = 0
+ self._reset_streaming_state()
+ return result
+ return StreamingParseResult(normal_text="", calls=calls)
+ except Exception as e:
+ logger.error(f"Error in parse_streaming_increment: {e}", exc_info=True)
+ return StreamingParseResult(normal_text=current_text)
+
+ def _parse_argument_pairs(
+ self, pairs: List[Tuple[str, str]], func_name: str, tools: List[Tool]
+ ) -> Dict[str, Any]:
+ arguments = {}
+ for arg_key, arg_value in pairs:
+ arg_key = arg_key.strip()
+ arg_type = get_argument_type_glm(func_name, arg_key, tools)
+ parsed_value, is_good_json = parse_arguments(arg_value, arg_type)
+ if arg_type == "string":
+ if isinstance(parsed_value, str):
+ arguments[arg_key] = parsed_value
+ elif isinstance(parsed_value, (dict, list)):
+ arguments[arg_key] = json.dumps(parsed_value, ensure_ascii=False)
+ else:
+ arguments[arg_key] = str(parsed_value)
+ elif arg_type is None:
+ arguments[arg_key] = parsed_value if is_good_json else arg_value
+ else:
+ arguments[arg_key] = parsed_value if is_good_json else arg_value
+ return arguments
+
+ def supports_structural_tag(self) -> bool:
+ return False
+
+ def structure_info(self) -> _GetInfoFunc:
+ raise NotImplementedError()
diff --git a/python/infinilm/agents/detectors/llama32_detector.py b/python/infinilm/agents/detectors/llama32_detector.py
new file mode 100644
index 000000000..e0dc2a289
--- /dev/null
+++ b/python/infinilm/agents/detectors/llama32_detector.py
@@ -0,0 +1,129 @@
+"""
+Detector for Llama 3.2 / 3.1 models with json tool call format.
+"""
+
+import json
+import logging
+import re
+from typing import List
+
+from infinilm.agents.base_detector import BaseFormatDetector
+from infinilm.agents.protocol import Tool
+from infinilm.agents.types import StreamingParseResult, StructureInfo, _GetInfoFunc
+from infinilm.agents.utils import safe_literal_eval
+
+logger = logging.getLogger(__name__)
+
+
+class Llama32Detector(BaseFormatDetector):
+ """
+ Detector for Llama 3.2 models with json tool call format.
+ Format Structure:
+ <|python_tag>{"name":"xxx", "arguments":{...}}
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.bot_token = "<|python_tag|>"
+ self.tool_call_separator = ";"
+
+ def _convert_python_dict_to_json(self, text: str) -> str:
+ """Convert Python dict strings to JSON format."""
+ try:
+ parsed = safe_literal_eval(text.strip())
+ if isinstance(parsed, dict):
+ return json.dumps(parsed, ensure_ascii=False)
+ except Exception:
+ pass
+ return text
+
+ def has_tool_call(self, text: str) -> bool:
+ return "<|python_tag|>" in text or text.startswith("{")
+
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ if "<|python_tag|>" not in text and not text.startswith("{"):
+ return StreamingParseResult(normal_text=text, calls=[])
+
+ if "<|python_tag|>" in text:
+ normal_text, action_text = text.split("<|python_tag|>", maxsplit=1)
+ else:
+ normal_text, action_text = "", text
+
+ decoder = json.JSONDecoder()
+ idx = 0
+ safe_idx = idx
+ all_actions = []
+ action_text_len = len(action_text)
+ while idx < action_text_len:
+ try:
+ obj, end = decoder.raw_decode(action_text[idx:])
+ all_actions.append(obj)
+ idx += end + len(self.tool_call_separator)
+ safe_idx = idx
+ except json.JSONDecodeError:
+ try:
+ dict_end = idx
+ brace_count = 0
+ for i in range(idx, action_text_len):
+ if action_text[i] == "{":
+ brace_count += 1
+ elif action_text[i] == "}":
+ brace_count -= 1
+ if brace_count == 0:
+ dict_end = i + 1
+ break
+ if dict_end > idx:
+ potential_dict = action_text[idx:dict_end]
+ json_version = self._convert_python_dict_to_json(potential_dict)
+ if json_version != potential_dict:
+ obj, _ = decoder.raw_decode(json_version)
+ all_actions.append(obj)
+ idx = dict_end + len(self.tool_call_separator)
+ safe_idx = idx
+ continue
+ except Exception:
+ pass
+ next_obj_start = action_text.find('{"name":', idx + 1)
+ if next_obj_start == -1:
+ break
+ idx = next_obj_start
+
+ # Objects without a "name" key are plain JSON answers, not tool
+ # calls; pass the original text through untouched.
+ if all_actions and not any(
+ isinstance(action, dict) and "name" in action for action in all_actions
+ ):
+ return StreamingParseResult(normal_text=text, calls=[])
+
+ calls = self.parse_base_json(all_actions, tools) if all_actions else []
+ trailing_text = (
+ action_text[safe_idx:].strip() if safe_idx < action_text_len else ""
+ )
+ return StreamingParseResult(
+ normal_text=normal_text + trailing_text, calls=calls
+ )
+
+ def parse_streaming_increment(
+ self, new_text: str, tools: List[Tool]
+ ) -> StreamingParseResult:
+ self._buffer += new_text
+ converted_buffer = self._buffer
+ converted_buffer = re.sub(r"'([^']*)':", r'"\1":', converted_buffer)
+ converted_buffer = re.sub(r":\s*'([^']*)'", r': "\1"', converted_buffer)
+ original_buffer = self._buffer
+ self._buffer = converted_buffer
+ try:
+ result = super().parse_streaming_increment("", tools)
+ return result
+ except Exception:
+ self._buffer = original_buffer
+ # original_buffer already contains new_text, so pass empty string
+ # to avoid adding new_text twice.
+ return super().parse_streaming_increment("", tools)
+
+ def structure_info(self) -> _GetInfoFunc:
+ return lambda name: StructureInfo(
+ begin='<|python_tag|>{"name":"' + name + '", "arguments":',
+ end="}",
+ trigger="<|python_tag|>",
+ )
diff --git a/python/infinilm/agents/detectors/qwen3_xml_detector.py b/python/infinilm/agents/detectors/qwen3_xml_detector.py
new file mode 100644
index 000000000..be11a6005
--- /dev/null
+++ b/python/infinilm/agents/detectors/qwen3_xml_detector.py
@@ -0,0 +1,264 @@
+"""
+Detector for Qwen3 models' xml tool call format.
+"""
+
+import json
+import logging
+import re
+from typing import List
+
+from infinilm.agents.base_detector import Allow, BaseFormatDetector
+from infinilm.agents.protocol import Tool
+from infinilm.agents.types import (
+ StreamingParseResult,
+ StructureInfo,
+ ToolCallItem,
+ _GetInfoFunc,
+)
+from infinilm.agents.utils import _find_common_prefix, _partial_json_loads
+
+logger = logging.getLogger(__name__)
+
+
+class Qwen3XmlDetector(BaseFormatDetector):
+ """
+ Detector for Qwen3 models with xml tool call format.
+ Format Structure:
+ {"name":"xxx", "arguments":{...}}
+ wrapped in opening/closing "tool_call" xml tags.
+
+ Streaming model: outside a block, text is emitted as normal content until
+ the opening tag is seen. Inside a block, content accumulates in a private
+ buffer; partial-JSON parsing streams the tool name and argument deltas as
+ they arrive, and the closing tag finalizes the call (guaranteeing the full
+ arguments are delivered exactly once).
+ """
+
+ def __init__(self):
+ super().__init__()
+ # Built via concatenation so the literal tag sequences stay intact
+ # in this source file.
+ self.bot_token = "<" + "tool_call" + ">"
+ self.eot_token = ""
+ self.tool_call_separator = "\n"
+ self._in_tool_block = False
+ self._block_buffer = ""
+
+ def clear(self):
+ super().clear()
+ self._in_tool_block = False
+ self._block_buffer = ""
+
+ def has_tool_call(self, text: str) -> bool:
+ return self.bot_token in text
+
+ @staticmethod
+ def _hold_len(value: str, token: str) -> int:
+ """How many trailing chars of ``value`` could be the start of ``token``."""
+ for i in range(min(len(value), len(token) - 1), 0, -1):
+ if value.endswith(token[:i]):
+ return i
+ return 0
+
+ def parse_streaming_increment(
+ self, new_text: str, tools: List[Tool]
+ ) -> StreamingParseResult:
+ self._buffer += new_text
+ tool_indices = self._get_tool_indices(tools)
+ calls = []
+ normal_parts: List[str] = []
+
+ while self._buffer or self._in_tool_block:
+ if not self._in_tool_block:
+ bot_idx = self._buffer.find(self.bot_token)
+ if bot_idx == -1:
+ # Hold back a trailing fragment that could start the tag.
+ hold = self._hold_len(self._buffer, self.bot_token)
+ if hold:
+ normal_parts.append(self._buffer[:-hold])
+ self._buffer = self._buffer[-hold:]
+ else:
+ normal_parts.append(self._buffer)
+ self._buffer = ""
+ # A stray closing tag outside a block is stripped.
+ emitted = "".join(normal_parts)
+ normal_parts = [emitted.replace(self.eot_token, "")]
+ break
+ # Emit text before the opening tag (strip stray closing tags,
+ # hold a trailing fragment that could start one).
+ before = self._buffer[:bot_idx]
+ eot_hold = self._hold_len(before, self.eot_token)
+ if eot_hold:
+ normal_parts.append(before[:-eot_hold])
+ self._buffer = before[-eot_hold:] + self._buffer[bot_idx:]
+ break
+ normal_parts.append(before.replace(self.eot_token, ""))
+ self._buffer = self._buffer[bot_idx + len(self.bot_token) :]
+ self._in_tool_block = True
+ self._block_buffer = ""
+ continue
+
+ # --- inside a tool block ---
+ self._block_buffer += self._buffer
+ self._buffer = ""
+
+ eot_idx = self._block_buffer.find(self.eot_token)
+ if eot_idx != -1:
+ calls.extend(self._finalize_block(eot_idx, tool_indices))
+ self._in_tool_block = False
+ continue
+
+ # Block not closed yet: stream name / argument deltas.
+ calls.extend(self._stream_partial_block(tool_indices))
+ break
+
+ normal_text = "".join(normal_parts)
+ return StreamingParseResult(normal_text=normal_text, calls=calls)
+
+ def _ensure_call_slot(self):
+ if self.current_tool_id == -1:
+ self.current_tool_id = len(self.prev_tool_call_arr)
+ self.streamed_args_for_tool.append("")
+ self.prev_tool_call_arr.append({})
+
+ def _stream_partial_block(self, tool_indices) -> List[ToolCallItem]:
+ """Emit name / argument deltas for a still-open block."""
+ calls: List[ToolCallItem] = []
+ flags = (
+ Allow.ALL
+ if (Allow is not None and self.current_tool_name_sent)
+ else (Allow.ALL & ~Allow.STR if Allow is not None else None)
+ )
+ try:
+ obj, _ = _partial_json_loads(self._block_buffer.strip(), flags)
+ except Exception:
+ return calls
+ if not isinstance(obj, dict):
+ return calls
+
+ name = obj.get("name")
+ if name and name in tool_indices and not self.current_tool_name_sent:
+ self._ensure_call_slot()
+ calls.append(self._call_item(self.current_tool_id, name, ""))
+ self.current_tool_name_sent = True
+
+ args = obj.get("arguments")
+ if args is not None and self.current_tool_name_sent:
+ self._ensure_call_slot()
+ cur_args_json = json.dumps(args, ensure_ascii=False)
+ prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get(
+ "arguments"
+ )
+ self.prev_tool_call_arr[self.current_tool_id]["arguments"] = args
+ if prev_arguments is not None:
+ prev_args_json = json.dumps(prev_arguments, ensure_ascii=False)
+ if cur_args_json != prev_args_json:
+ sent = self.streamed_args_for_tool[self.current_tool_id]
+ prefix = _find_common_prefix(prev_args_json, cur_args_json)
+ diff = prefix[len(sent) :]
+ if diff:
+ self.streamed_args_for_tool[self.current_tool_id] += diff
+ calls.append(
+ self._call_item(
+ self.current_tool_id,
+ None,
+ diff,
+ )
+ )
+ return calls
+
+ def _finalize_block(self, eot_idx: int, tool_indices) -> List[ToolCallItem]:
+ """Close the block at ``eot_idx`` and deliver any unsent remainder."""
+ calls: List[ToolCallItem] = []
+ content = self._block_buffer[:eot_idx].strip()
+ remainder = self._block_buffer[eot_idx + len(self.eot_token) :]
+ self._block_buffer = ""
+
+ try:
+ obj = json.loads(content)
+ except json.JSONDecodeError:
+ obj = None
+
+ if isinstance(obj, dict):
+ name = obj.get("name")
+ if name and name in tool_indices:
+ if not self.current_tool_name_sent:
+ self._ensure_call_slot()
+ calls.append(self._call_item(self.current_tool_id, name, ""))
+ self.current_tool_name_sent = True
+ args = obj.get("arguments") or obj.get("parameters") or {}
+ final_args_json = json.dumps(args, ensure_ascii=False)
+ sent = self.streamed_args_for_tool[self.current_tool_id]
+ if final_args_json.startswith(sent):
+ diff = final_args_json[len(sent) :]
+ else:
+ # Snapshots diverged from what was streamed; re-emit the
+ # full arguments to guarantee a correct, complete value.
+ diff = final_args_json
+ sent = ""
+ if diff:
+ self.streamed_args_for_tool[self.current_tool_id] = sent + diff
+ calls.append(self._call_item(self.current_tool_id, None, diff))
+
+ # Reset per-call state for the next block; reprocess any remainder.
+ self.current_tool_name_sent = False
+ self.current_tool_id = -1
+ self._buffer = remainder + self._buffer
+ return calls
+
+ @staticmethod
+ def _call_item(tool_index: int, name, parameters: str) -> ToolCallItem:
+ return ToolCallItem(tool_index=tool_index, name=name, parameters=parameters)
+
+ def finish(self, tools: List[Tool]) -> StreamingParseResult:
+ """Flush buffered state; release a truncated block as normal text."""
+ result = StreamingParseResult()
+ sp_result = self.parse_streaming_increment("", tools)
+ result.normal_text += sp_result.normal_text
+ result.calls.extend(sp_result.calls)
+
+ leftover = ""
+ if self._in_tool_block and self._block_buffer:
+ # Truncated block at end of stream: emit it verbatim.
+ leftover = self.bot_token + self._block_buffer
+ self._block_buffer = ""
+ self._in_tool_block = False
+ if self._buffer:
+ leftover += self._buffer
+ self._buffer = ""
+ if leftover:
+ result.normal_text += leftover
+ return result
+
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ """Parse all complete tool_call blocks in one shot."""
+ pattern = re.compile(
+ re.escape(self.bot_token) + r"(.*?)" + re.escape(self.eot_token),
+ re.DOTALL,
+ )
+ matches = pattern.findall(text)
+ if not matches:
+ return StreamingParseResult(normal_text=text, calls=[])
+
+ normal_text = pattern.sub("", text).strip()
+ calls = []
+ for content in matches:
+ try:
+ action = json.loads(content.strip())
+ except json.JSONDecodeError:
+ continue
+ calls.extend(self.parse_base_json(action, tools, len(calls)))
+ return StreamingParseResult(normal_text=normal_text, calls=calls)
+
+ def structure_info(self) -> _GetInfoFunc:
+ """Return a builder for Qwen3 XML tool-call structural tags.
+
+ The tag bounds are ``...``. This metadata
+ is ready for XGrammar / constrained-decoding integration; the tag
+ is not yet enforced because MVP disables ``get_structural_tag()``.
+ """
+ return lambda name: StructureInfo(
+ begin="",
+ end="",
+ trigger="",
+ )
diff --git a/python/infinilm/agents/function_call_parser.py b/python/infinilm/agents/function_call_parser.py
new file mode 100644
index 000000000..7b79573e4
--- /dev/null
+++ b/python/infinilm/agents/function_call_parser.py
@@ -0,0 +1,154 @@
+"""
+Parser for function/tool calls in model outputs.
+Supports GLM-4 and Llama-3.1/3.2 tool call formats.
+"""
+
+import inspect
+import logging
+from typing import Dict, List, Optional, Tuple, Type
+
+from infinilm.agents.base_detector import BaseFormatDetector
+from infinilm.agents.detectors import (
+ Glm4Chat0414Detector,
+ Glm4MoeDetector,
+ Llama32Detector,
+ Qwen3XmlDetector,
+)
+from infinilm.agents.protocol import Function, Tool
+from infinilm.agents.types import StreamingParseResult, ToolCallItem
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_tools(tools: Optional[list]) -> List[Tool]:
+ """Normalize raw dict/list tools into Tool model instances."""
+ if not tools:
+ return []
+ normalized: List[Tool] = []
+ for t in tools:
+ if isinstance(t, Tool):
+ normalized.append(t)
+ elif isinstance(t, dict):
+ function_data = t.get("function", {})
+ function = Function(
+ description=function_data.get("description"),
+ name=function_data.get("name", ""),
+ parameters=function_data.get("parameters"),
+ strict=function_data.get("strict"),
+ )
+ normalized.append(Tool(type=t.get("type", "function"), function=function))
+ else:
+ raise ValueError(f"Invalid tool type: {type(t)}")
+ return normalized
+
+
+class FunctionCallParser:
+ """
+ Parser for function/tool calls in model outputs.
+ Handles both streaming and non-streaming parsing using a detector.
+ """
+
+ ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = {
+ "glm": Glm4MoeDetector,
+ "glm45": Glm4MoeDetector,
+ "glm47": Glm4MoeDetector,
+ # GLM-4-9B-Chat-0414 "metadata" format: function name on one line,
+ # JSON arguments on the next line (no xml tags).
+ "glm4": Glm4Chat0414Detector,
+ "glm49b": Glm4Chat0414Detector,
+ "glm4-9b-0414": Glm4Chat0414Detector,
+ "glm-4-9b-0414": Glm4Chat0414Detector,
+ "llama3": Llama32Detector,
+ "llama32": Llama32Detector,
+ "llama31": Llama32Detector,
+ # Qwen3 xml format: {"name": "...", "arguments": {...}}
+ "qwen3": Qwen3XmlDetector,
+ "qwen3-30b-a3b": Qwen3XmlDetector,
+ }
+
+ def __init__(
+ self, tool_call_parser: str, tools: Optional[list] = None, tokenizer=None
+ ):
+ detector_class = self.ToolCallParserEnum.get(tool_call_parser)
+ if detector_class:
+ kwargs = {}
+ if tokenizer is not None:
+ sig = inspect.signature(detector_class)
+ if "tokenizer" in sig.parameters:
+ kwargs["tokenizer"] = tokenizer
+ detector = detector_class(**kwargs)
+ else:
+ raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")
+
+ self.detector = detector
+ self.tools = _normalize_tools(tools)
+
+ def _ensure_tools(self, tools: Optional[list] = None) -> List[Tool]:
+ if tools is not None:
+ return _normalize_tools(tools)
+ return self.tools
+
+ def has_tool_call(self, text: str) -> bool:
+ if not self.tools:
+ return False
+ return self.detector.has_tool_call(text)
+
+ def parse_non_stream(
+ self, full_text: str, tools: Optional[list] = None
+ ) -> Tuple[str, list[ToolCallItem]]:
+ tools = self._ensure_tools(tools)
+ if not tools:
+ return full_text, []
+ has_tool_call = self.detector.has_tool_call(full_text)
+ parsed_result = self.detector.detect_and_parse(full_text, tools)
+ tool_call_list = parsed_result.calls
+ if tool_call_list or has_tool_call:
+ return parsed_result.normal_text, tool_call_list
+ else:
+ return full_text, []
+
+ def parse_streaming_increment(
+ self, text: str, delta_text: str, tools: Optional[list] = None
+ ) -> StreamingParseResult:
+ """Streaming increment wrapper; delegates to detector.
+
+ Args:
+ text: Accumulated text so far (kept for API compatibility).
+ delta_text: New text chunk from this streaming step.
+ tools: Optional override list of tools.
+ """
+ tools = self._ensure_tools(tools)
+ if not tools:
+ return StreamingParseResult(normal_text=delta_text)
+ sp_result = self.detector.parse_streaming_increment(delta_text, tools)
+ return sp_result
+
+ def parse_stream_chunk(self, chunk_text: str) -> Tuple[str, list[ToolCallItem]]:
+ if not self.tools:
+ return chunk_text, []
+ final_normal_text = ""
+ final_calls = []
+
+ sp_result = self.detector.parse_streaming_increment(chunk_text, self.tools)
+ if sp_result.normal_text:
+ final_normal_text = sp_result.normal_text
+ if sp_result.calls:
+ final_calls.extend(sp_result.calls)
+ final_normal_text = sp_result.normal_text
+
+ return final_normal_text, final_calls
+
+ def parse_stream_end(
+ self, tools: Optional[list] = None
+ ) -> Tuple[str, list[ToolCallItem]]:
+ """Flush any buffered state at the end of a stream.
+
+ Args:
+ tools: Optional override list of tools; falls back to the tools
+ bound at construction time.
+ """
+ tools = self._ensure_tools(tools)
+ if not tools:
+ return "", []
+ sp_result = self.detector.finish(tools)
+ return sp_result.normal_text, sp_result.calls
diff --git a/python/infinilm/agents/message_adapter.py b/python/infinilm/agents/message_adapter.py
new file mode 100644
index 000000000..29a394ab5
--- /dev/null
+++ b/python/infinilm/agents/message_adapter.py
@@ -0,0 +1,79 @@
+"""
+Adapt incoming chat requests to model-family tool-call conventions.
+"""
+
+from typing import Optional
+
+# Tool-call parsers whose models use the GLM-4 "metadata" convention:
+# assistant tool calls are rendered via the message ``metadata`` field and
+# tool results use the ``observation`` role (see the GLM-4-9B-Chat-0414
+# chat template). OpenAI-style histories must be rewritten accordingly,
+# otherwise the template silently drops them and the tool loop cannot close.
+GLM4_METADATA_PARSER_ALIASES = {
+ "glm4",
+ "glm49b",
+ "glm4-9b-0414",
+ "glm-4-9b-0414",
+}
+
+# Qwen3 models natively understand OpenAI-format tool messages in their
+# chat templates, so no rewriting is required.
+QWEN3_PARSER_ALIASES = {
+ "qwen3",
+ "qwen3-30b-a3b",
+}
+
+
+def adapt_messages(messages: list, tool_call_parser: Optional[str]) -> list:
+ """Rewrite OpenAI-format tool history for parsers that need it.
+
+ Applies only to the GLM-4 metadata-style convention
+ (``GLM4_METADATA_PARSER_ALIASES``): ``tool`` messages become
+ ``observation`` messages, and assistant messages carrying
+ ``tool_calls`` become one ``metadata`` message per call. All other
+ requests pass through unchanged.
+ """
+ if tool_call_parser not in GLM4_METADATA_PARSER_ALIASES:
+ # Qwen3, Llama and other parsers pass OpenAI-format messages straight
+ # through; their chat templates understand the standard roles.
+ return messages
+
+ adapted = []
+ for msg in messages:
+ if not isinstance(msg, dict):
+ adapted.append(msg)
+ continue
+ if msg.get("role") == "tool":
+ adapted.append({**msg, "role": "observation"})
+ elif msg.get("role") == "assistant" and msg.get("tool_calls"):
+ if msg.get("content"):
+ adapted.append({"role": "assistant", "content": msg["content"]})
+ for tc in msg.get("tool_calls") or []:
+ fn = tc.get("function", {}) if isinstance(tc, dict) else {}
+ adapted.append(
+ {
+ "role": "assistant",
+ "metadata": fn.get("name", ""),
+ "content": fn.get("arguments", "") or "{}",
+ }
+ )
+ else:
+ adapted.append(msg)
+ return adapted
+
+
+def prepare_chat_template_kwargs(data: dict) -> None:
+ """Pack tool definitions from the request into ``chat_template_kwargs``.
+
+ Pops nothing: ``tools``/``tool_choice`` stay in ``data`` for the output
+ post-processing, and are additionally forwarded to ``apply_chat_template``
+ so the model prompt actually sees the tool definitions.
+ """
+ tools = data.get("tools") or []
+ tool_choice = data.get("tool_choice", "auto")
+ chat_template_kwargs = data.get("chat_template_kwargs") or {}
+ if tools:
+ chat_template_kwargs["tools"] = tools
+ if tool_choice:
+ chat_template_kwargs["tool_choice"] = tool_choice
+ data["chat_template_kwargs"] = chat_template_kwargs
diff --git a/python/infinilm/agents/protocol.py b/python/infinilm/agents/protocol.py
new file mode 100644
index 000000000..a8191d89a
--- /dev/null
+++ b/python/infinilm/agents/protocol.py
@@ -0,0 +1,131 @@
+"""
+OpenAI-compatible protocol models and response builders for agent support.
+Minimal subset needed for tools, tool_choice, and reasoning_content.
+"""
+
+import time
+from typing import Any, Dict, List, Literal, Optional
+
+from pydantic import BaseModel
+
+
+class Function(BaseModel):
+ description: Optional[str] = None
+ name: str
+ parameters: Optional[Dict[str, Any]] = None
+ strict: Optional[bool] = None
+
+
+class Tool(BaseModel):
+ type: Literal["function"] = "function"
+ function: Function
+
+
+class ToolChoiceFuncName(BaseModel):
+ name: str
+
+
+class ToolChoice(BaseModel):
+ type: Literal["function"] = "function"
+ function: ToolChoiceFuncName
+
+
+class FunctionResponse(BaseModel):
+ name: str
+ arguments: str
+
+
+class ChatCompletionMessageToolCall(BaseModel):
+ id: str
+ type: Literal["function"] = "function"
+ function: FunctionResponse
+
+
+class DeltaMessage(BaseModel):
+ role: Optional[str] = None
+ content: Optional[str] = None
+ reasoning_content: Optional[str] = None
+ tool_calls: Optional[List[ChatCompletionMessageToolCall]] = None
+
+
+def chunk_json(
+ id_,
+ content=None,
+ role=None,
+ finish_reason=None,
+ model: str = "unknown",
+ reasoning_content=None,
+ tool_calls=None,
+ usage=None,
+):
+ """Generate JSON chunk for streaming response."""
+ delta = {}
+ if content is not None:
+ delta["content"] = content
+ if role:
+ delta["role"] = role
+ if reasoning_content is not None:
+ delta["reasoning_content"] = reasoning_content
+ if tool_calls:
+ delta["tool_calls"] = tool_calls
+ chunk = {
+ "id": id_,
+ "object": "chat.completion.chunk",
+ "created": int(time.time()),
+ "model": model,
+ "system_fingerprint": None,
+ "choices": [
+ {
+ "index": 0,
+ "delta": delta,
+ "logprobs": None,
+ "finish_reason": finish_reason,
+ }
+ ],
+ }
+ if usage:
+ chunk["usage"] = usage
+ return chunk
+
+
+def completion_json(
+ id_,
+ content,
+ role="assistant",
+ finish_reason="stop",
+ model: str = "unknown",
+ prompt_tokens: int = 0,
+ completion_tokens: int = 0,
+ total_tokens: int = 0,
+ reasoning_content=None,
+ tool_calls=None,
+):
+ """Generate JSON response for non-streaming completion."""
+ message = {
+ "role": role,
+ "content": content,
+ }
+ if reasoning_content is not None:
+ message["reasoning_content"] = reasoning_content
+ if tool_calls:
+ message["tool_calls"] = tool_calls
+ return {
+ "id": id_,
+ "object": "chat.completion",
+ "created": int(time.time()),
+ "model": model,
+ "system_fingerprint": None,
+ "choices": [
+ {
+ "index": 0,
+ "message": message,
+ "logprobs": None,
+ "finish_reason": finish_reason,
+ }
+ ],
+ "usage": {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": total_tokens,
+ },
+ }
diff --git a/python/infinilm/agents/reasoning_parser.py b/python/infinilm/agents/reasoning_parser.py
new file mode 100644
index 000000000..f8446f45b
--- /dev/null
+++ b/python/infinilm/agents/reasoning_parser.py
@@ -0,0 +1,271 @@
+"""
+Parser for reasoning/thinking content extraction.
+"""
+
+from typing import List, Optional, Tuple
+
+
+class ReasoningStreamingParseResult:
+ """Result of reasoning content parsing."""
+
+ def __init__(
+ self, reasoning_content: str = "", normal_text: str = "", complete: bool = False
+ ):
+ self.reasoning_content = reasoning_content
+ self.normal_text = normal_text
+ self.complete = complete
+
+ def __repr__(self):
+ return (
+ f"ReasoningStreamingParseResult("
+ f"reasoning_content={self.reasoning_content!r}, "
+ f"normal_text={self.normal_text!r}, "
+ f"complete={self.complete})"
+ )
+
+
+class BaseReasoningFormatDetector:
+ """Abstract base for reasoning format detectors."""
+
+ def __init__(
+ self,
+ start_token: str,
+ end_token: str,
+ stream_start_prefill: bool = False,
+ include_start_token: bool = False,
+ include_end_token: bool = False,
+ starts_with_start_token: bool = True,
+ is_full_suffix: bool = True,
+ ):
+ self.start_token = start_token
+ self.end_token = end_token
+ self.stream_start_prefill = stream_start_prefill
+ self.include_start_token = include_start_token
+ self.include_end_token = include_end_token
+ self.starts_with_start_token = starts_with_start_token
+ self.is_full_suffix = is_full_suffix
+ self._buffer = ""
+ self.found_reasoning_end = False
+ self.reasoning_started = stream_start_prefill
+
+ def clear(self):
+ self._buffer = ""
+ self.found_reasoning_end = False
+ self.reasoning_started = self.stream_start_prefill
+
+ def detect_and_parse(self, text: str) -> Tuple[ReasoningStreamingParseResult, ...]:
+ return self._detect_and_parse_impl(text)
+
+ def parse_streaming_increment(
+ self, text: str, delta_text: str
+ ) -> ReasoningStreamingParseResult:
+ return self._parse_streaming_increment_impl(text, delta_text)
+
+ def _detect_and_parse_impl(
+ self, text: str
+ ) -> Tuple[ReasoningStreamingParseResult, ...]:
+ if self.start_token not in text:
+ if self.stream_start_prefill:
+ if self.end_token in text:
+ end_idx = text.index(self.end_token)
+ reasoning = text[:end_idx]
+ normal = text[end_idx + len(self.end_token) :]
+ return (
+ ReasoningStreamingParseResult(
+ reasoning_content=reasoning,
+ normal_text=normal,
+ complete=True,
+ ),
+ )
+ return (
+ ReasoningStreamingParseResult(
+ reasoning_content=text,
+ normal_text="",
+ complete=False,
+ ),
+ )
+ return (
+ ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=text, complete=True
+ ),
+ )
+ start_idx = text.index(self.start_token)
+ after_start = text[start_idx + len(self.start_token) :]
+ if self.end_token not in after_start:
+ reasoning = after_start
+ normal = ""
+ return (
+ ReasoningStreamingParseResult(
+ reasoning_content=reasoning, normal_text=normal, complete=False
+ ),
+ )
+ end_idx = after_start.index(self.end_token)
+ reasoning = after_start[:end_idx]
+ normal = after_start[end_idx + len(self.end_token) :]
+ return (
+ ReasoningStreamingParseResult(
+ reasoning_content=reasoning, normal_text="", complete=True
+ ),
+ ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=normal, complete=True
+ ),
+ )
+
+ def _parse_streaming_increment_impl(
+ self, text: str, delta_text: str
+ ) -> ReasoningStreamingParseResult:
+ self._buffer += delta_text
+ if not self.reasoning_started:
+ if self.start_token in self._buffer:
+ self.reasoning_started = True
+ idx = self._buffer.index(self.start_token)
+ normal_before = self._buffer[:idx]
+ self._buffer = self._buffer[idx + len(self.start_token) :]
+ out = self._buffer
+ self._buffer = ""
+ return ReasoningStreamingParseResult(
+ reasoning_content=out, normal_text=normal_before, complete=False
+ )
+ if len(self._buffer) >= len(self.start_token):
+ for i in range(1, len(self.start_token)):
+ if self._buffer.endswith(self.start_token[:i]):
+ out = self._buffer[:-i]
+ self._buffer = self._buffer[-i:]
+ if out:
+ return ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=out, complete=True
+ )
+ return ReasoningStreamingParseResult()
+ out = self._buffer
+ self._buffer = ""
+ return ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=out, complete=True
+ )
+ return ReasoningStreamingParseResult()
+ if not self.found_reasoning_end:
+ if self.end_token in self._buffer:
+ self.found_reasoning_end = True
+ idx = self._buffer.index(self.end_token)
+ reasoning = self._buffer[:idx]
+ normal = self._buffer[idx + len(self.end_token) :]
+ self._buffer = ""
+ return ReasoningStreamingParseResult(
+ reasoning_content=reasoning, normal_text=normal, complete=True
+ )
+ if len(self._buffer) > len(self.end_token):
+ reasoning = self._buffer[: -len(self.end_token)]
+ self._buffer = self._buffer[-len(self.end_token) :]
+ return ReasoningStreamingParseResult(
+ reasoning_content=reasoning, normal_text="", complete=False
+ )
+ return ReasoningStreamingParseResult()
+ normal = self._buffer
+ self._buffer = ""
+ return ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=normal, complete=True
+ )
+
+
+class Glm45Detector(BaseReasoningFormatDetector):
+ """Detector for GLM-4.5 thinking tags (PUA unicode start/end tokens)."""
+
+ def __init__(self, **kwargs):
+ super().__init__(
+ start_token="think",
+ end_token="/think",
+ stream_start_prefill=True,
+ include_start_token=False,
+ include_end_token=False,
+ **kwargs,
+ )
+
+
+class ThinkTagDetector(BaseReasoningFormatDetector):
+ """Generic detector for standard 'thinking' / 'response' reasoning tags."""
+
+ def __init__(self, **kwargs):
+ super().__init__(
+ start_token="",
+ end_token="",
+ stream_start_prefill=False,
+ include_start_token=False,
+ include_end_token=False,
+ **kwargs,
+ )
+
+
+class DeepSeekR1Detector(BaseReasoningFormatDetector):
+ """Detector for DeepSeek-R1 / QwQ style ... reasoning tags."""
+
+ def __init__(self, **kwargs):
+ super().__init__(
+ start_token="",
+ end_token="",
+ stream_start_prefill=False,
+ include_start_token=False,
+ include_end_token=False,
+ **kwargs,
+ )
+
+
+class ReasoningParser:
+ """Parses reasoning/thinking content from LLM outputs."""
+
+ def __init__(self, reasoning_parser_name: Optional[str] = None):
+ self.reasoning_parser_name = reasoning_parser_name
+ self.format_detectors: List[BaseReasoningFormatDetector] = []
+ if reasoning_parser_name:
+ name = reasoning_parser_name.lower()
+ if name in {
+ "glm4",
+ "glm45",
+ "glm-4",
+ "glm-4.5",
+ "glm-4.5-air",
+ "glm-4.5-flash",
+ }:
+ self.format_detectors = [Glm45Detector()]
+ elif name in {"think", "thinking"}:
+ self.format_detectors = [ThinkTagDetector()]
+ elif name in {
+ "deepseek",
+ "deepseek-r1",
+ "deepseek_r1",
+ "qwq",
+ "qwq-32b",
+ "qwen3",
+ "qwen3-thinking",
+ }:
+ # DeepSeek-R1, QwQ and Qwen3 thinking models use the short ... tags,
+ # not ....
+ self.format_detectors = [DeepSeekR1Detector()]
+ else:
+ self.format_detectors = []
+ else:
+ self.format_detectors = []
+
+ def clear(self):
+ for detector in self.format_detectors:
+ detector.clear()
+
+ def extract_reasoning_content_streaming(
+ self, previous_text: str, current_text: str, delta_text: str
+ ) -> ReasoningStreamingParseResult:
+ if not self.format_detectors:
+ return ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=delta_text, complete=True
+ )
+ return self.format_detectors[0].parse_streaming_increment(
+ current_text, delta_text
+ )
+
+ def extract_reasoning_content(
+ self, text: str
+ ) -> Tuple[ReasoningStreamingParseResult, ...]:
+ if not self.format_detectors:
+ return (
+ ReasoningStreamingParseResult(
+ reasoning_content="", normal_text=text, complete=True
+ ),
+ )
+ return self.format_detectors[0].detect_and_parse(text)
diff --git a/python/infinilm/agents/stream_parser.py b/python/infinilm/agents/stream_parser.py
new file mode 100644
index 000000000..1bdaed036
--- /dev/null
+++ b/python/infinilm/agents/stream_parser.py
@@ -0,0 +1,197 @@
+"""
+Per-request parsing of generated text into agent-style output.
+
+The model forward pass produces plain text tokens. These helpers convert that
+text stream into the chat-completion protocol fields (``reasoning_content``,
+``content`` and OpenAI-format ``tool_calls``), so all parser state and
+protocol formatting stays inside ``infinilm.agents`` instead of being spread
+across the HTTP layer.
+"""
+
+import json
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+from infinilm.agents.function_call_parser import FunctionCallParser
+from infinilm.agents.protocol import chunk_json
+from infinilm.agents.reasoning_parser import ReasoningParser
+from infinilm.agents.types import ToolCallItem
+
+
+@dataclass
+class AgentDelta:
+ """One increment of parsed model output, ready for the API protocol."""
+
+ reasoning_content: str = ""
+ content: str = ""
+ tool_calls: List[Dict] = field(default_factory=list)
+
+
+def format_streaming_tool_calls(calls: List[ToolCallItem]) -> List[Dict]:
+ """Format parsed calls as OpenAI streaming ``delta.tool_calls`` items."""
+ return [
+ {
+ "index": call.tool_index,
+ "id": f"call_{call.tool_index}",
+ "type": "function",
+ "function": {
+ "name": call.name or "",
+ "arguments": call.parameters,
+ },
+ }
+ for call in calls
+ ]
+
+
+def format_tool_calls(calls: List[ToolCallItem]) -> List[Dict]:
+ """Format parsed calls as OpenAI ``message.tool_calls`` items."""
+ return [
+ {
+ "id": f"call_{index}",
+ "type": "function",
+ "function": {
+ "name": call.name or "",
+ "arguments": call.parameters,
+ },
+ }
+ for index, call in enumerate(calls)
+ ]
+
+
+class AgentStreamParser:
+ """Stateful per-request parser for a streamed generation.
+
+ Splits each generated token increment into reasoning content, visible
+ content and tool-call deltas. Create one instance per request; instances
+ must not be shared across concurrent streams.
+ """
+
+ def __init__(
+ self,
+ tool_call_parser: Optional[str] = None,
+ reasoning_parser: Optional[str] = None,
+ tools: Optional[list] = None,
+ ):
+ self._reasoning_parser = (
+ ReasoningParser(reasoning_parser_name=reasoning_parser)
+ if reasoning_parser
+ else None
+ )
+ self._tool_call_parser = (
+ FunctionCallParser(tool_call_parser=tool_call_parser, tools=tools or [])
+ if tool_call_parser
+ else None
+ )
+ self._text = ""
+ self.has_tool_calls = False
+
+ def process_delta(self, token_text: str) -> AgentDelta:
+ """Parse one generated token increment."""
+ self._text += token_text
+ delta_reasoning = ""
+ delta_normal = token_text
+
+ if self._reasoning_parser:
+ previous_text = self._text[: -len(token_text)] if token_text else self._text
+ result = self._reasoning_parser.extract_reasoning_content_streaming(
+ previous_text=previous_text,
+ current_text=self._text,
+ delta_text=token_text,
+ )
+ delta_reasoning = result.reasoning_content or ""
+ delta_normal = result.normal_text or ""
+
+ delta_content = delta_normal
+ tool_calls: List[Dict] = []
+ if self._tool_call_parser and delta_normal:
+ result = self._tool_call_parser.parse_streaming_increment(
+ self._text, delta_normal
+ )
+ delta_content = result.normal_text
+ if result.calls:
+ tool_calls = format_streaming_tool_calls(result.calls)
+ self.has_tool_calls = True
+
+ return AgentDelta(
+ reasoning_content=delta_reasoning,
+ content=delta_content,
+ tool_calls=tool_calls,
+ )
+
+ def flush(self) -> AgentDelta:
+ """Flush buffered state at the end of the stream."""
+ if not self._tool_call_parser:
+ return AgentDelta()
+ normal_text, calls = self._tool_call_parser.parse_stream_end()
+ if calls:
+ self.has_tool_calls = True
+ return AgentDelta(
+ content=normal_text,
+ tool_calls=format_streaming_tool_calls(calls),
+ )
+
+ def delta_events(self, token_text: str, request_id: str, model: str) -> List[str]:
+ """Parse one token increment and render it as OpenAI SSE lines.
+
+ Returns an empty list when the increment carries nothing to emit.
+ """
+ return delta_to_sse_chunks(request_id, model, self.process_delta(token_text))
+
+ def flush_events(self, request_id: str, model: str) -> List[str]:
+ """Flush buffered state and render it as OpenAI SSE lines."""
+ return delta_to_sse_chunks(request_id, model, self.flush())
+
+
+def delta_to_sse_chunks(request_id: str, model: str, delta: AgentDelta) -> List[str]:
+ """Render an ``AgentDelta`` as OpenAI streaming SSE ``data:`` lines."""
+ if not (delta.reasoning_content or delta.content or delta.tool_calls):
+ return []
+ chunk = chunk_json(
+ request_id,
+ content=delta.content or None,
+ reasoning_content=delta.reasoning_content or None,
+ tool_calls=delta.tool_calls or None,
+ model=model,
+ )
+ return [f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n"]
+
+
+def parse_full_response(
+ text: str,
+ tool_call_parser: Optional[str] = None,
+ reasoning_parser: Optional[str] = None,
+ tools: Optional[list] = None,
+) -> Tuple[Optional[str], str, List[Dict]]:
+ """One-shot parse of a complete (non-streaming) response.
+
+ Returns ``(reasoning_content, content, tool_calls)`` where
+ ``tool_calls`` is a list of OpenAI-format tool-call dicts.
+ """
+ reasoning_content: Optional[str] = None
+ normal_text = text
+
+ if reasoning_parser:
+ results = ReasoningParser(
+ reasoning_parser_name=reasoning_parser
+ ).extract_reasoning_content(text)
+ if len(results) >= 2:
+ reasoning_content = results[0].reasoning_content or None
+ normal_text = results[1].normal_text or ""
+ elif len(results) == 1:
+ if results[0].reasoning_content:
+ reasoning_content = results[0].reasoning_content or None
+ normal_text = results[0].normal_text or ""
+ else:
+ normal_text = results[0].normal_text or text
+
+ tool_calls: List[Dict] = []
+ if tool_call_parser:
+ parser = FunctionCallParser(
+ tool_call_parser=tool_call_parser, tools=tools or []
+ )
+ normal_text_after, call_list = parser.parse_non_stream(normal_text)
+ if call_list:
+ tool_calls = format_tool_calls(call_list)
+ normal_text = normal_text_after
+
+ return reasoning_content, normal_text, tool_calls
diff --git a/python/infinilm/agents/types.py b/python/infinilm/agents/types.py
new file mode 100644
index 000000000..e8433e459
--- /dev/null
+++ b/python/infinilm/agents/types.py
@@ -0,0 +1,38 @@
+"""
+Core types for agent/tool-call parsing.
+"""
+
+from dataclasses import dataclass
+from typing import Callable, List, Optional
+
+from pydantic import BaseModel
+
+
+class ToolCallItem(BaseModel):
+ """Simple encapsulation of the parsed ToolCall result for easier usage in streaming contexts."""
+
+ tool_index: int
+ name: Optional[str] = None
+ parameters: str = "" # JSON string
+
+
+class StreamingParseResult(BaseModel):
+ """Result of streaming incremental parsing."""
+
+ normal_text: str = ""
+ calls: List[ToolCallItem] = []
+
+
+@dataclass
+class StructureInfo:
+ begin: str
+ end: str
+ trigger: str
+
+
+"""
+Helper alias of function
+Usually it is a function that takes a name string and returns a StructureInfo object,
+which can be used to construct a structural_tag object
+"""
+_GetInfoFunc = Callable[[str], StructureInfo]
diff --git a/python/infinilm/agents/utils.py b/python/infinilm/agents/utils.py
new file mode 100644
index 000000000..390761e1e
--- /dev/null
+++ b/python/infinilm/agents/utils.py
@@ -0,0 +1,229 @@
+"""
+Utility functions for agent parsing.
+"""
+
+import ast
+import json
+import logging
+import threading
+import warnings
+from json import JSONDecodeError, JSONDecoder
+from json.decoder import WHITESPACE
+from typing import Any, Dict, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+try:
+ import partial_json_parser
+ from partial_json_parser.core.options import Allow
+
+ PARTIAL_JSON_AVAILABLE = True
+except ImportError:
+ partial_json_parser = None
+ Allow = None
+ PARTIAL_JSON_AVAILABLE = False
+
+
+def _find_common_prefix(s1: str, s2: str) -> str:
+ prefix = ""
+ min_length = min(len(s1), len(s2))
+ for i in range(0, min_length):
+ if s1[i] == s2[i]:
+ prefix += s1[i]
+ else:
+ break
+ return prefix
+
+
+def _partial_json_loads(input_str: str, flags: Any) -> Tuple[Any, int]:
+ """
+ Parse incomplete or partial JSON strings commonly encountered during streaming.
+ Falls back to standard JSONDecoder if partial_json_parser is unavailable.
+ """
+ if PARTIAL_JSON_AVAILABLE and partial_json_parser is not None:
+ try:
+ return (partial_json_parser.loads(input_str, flags), len(input_str))
+ except (JSONDecodeError, IndexError) as e:
+ msg = getattr(e, "msg", str(e))
+ if "Extra data" in msg or "pop from empty list" in msg:
+ start = WHITESPACE.match(input_str, 0).end()
+ obj, end = JSONDecoder().raw_decode(input_str, start)
+ return obj, end
+ raise
+ except AssertionError as e:
+ raise JSONDecodeError(
+ "partial_json_parser assertion (treat as incomplete)", input_str, 0
+ ) from e
+ else:
+ # Fallback: use standard JSONDecoder which handles complete JSON only
+ # In streaming context, this will raise on incomplete JSON, which is expected
+ start = WHITESPACE.match(input_str, 0).end()
+ obj, end = JSONDecoder().raw_decode(input_str, start)
+ return obj, end
+
+
+def _is_complete_json(input_str: str) -> bool:
+ try:
+ json.loads(input_str)
+ return True
+ except JSONDecodeError:
+ return False
+
+
+_safe_ast_lock = threading.Lock()
+
+
+def _run_ast_quiet(fn, *args):
+ with _safe_ast_lock, warnings.catch_warnings():
+ warnings.filterwarnings("ignore", category=SyntaxWarning)
+ warnings.filterwarnings("ignore", category=DeprecationWarning)
+ return fn(*args)
+
+
+def safe_literal_eval(value: str) -> Any:
+ return _run_ast_quiet(ast.literal_eval, value)
+
+
+def get_schema_properties(schema: Any) -> Dict[str, Any]:
+ """Top-level ``properties`` of a tool ``parameters`` schema."""
+ if not isinstance(schema, dict):
+ return {}
+ properties = schema.get("properties")
+ if isinstance(properties, dict):
+ return properties
+ merged: Dict[str, Any] = {}
+ for keyword in ("anyOf", "oneOf", "allOf"):
+ branches = schema.get(keyword)
+ if isinstance(branches, list):
+ for branch in branches:
+ for key, value in get_schema_properties(branch).items():
+ merged.setdefault(key, value)
+ return merged
+
+
+def infer_type_from_json_schema(schema: Dict[str, Any]) -> Optional[str]:
+ """Infer the primary type of a parameter from JSON Schema."""
+ if not isinstance(schema, dict):
+ return None
+
+ if "type" in schema:
+ type_value = schema["type"]
+ if isinstance(type_value, str):
+ return type_value
+ elif isinstance(type_value, list) and type_value:
+ non_null_types = [t for t in type_value if t != "null"]
+ if non_null_types:
+ return non_null_types[0]
+ return "string"
+
+ if "anyOf" in schema or "oneOf" in schema:
+ schemas = schema.get("anyOf") or schema.get("oneOf")
+ types = []
+ if isinstance(schemas, list):
+ for sub_schema in schemas:
+ inferred_type = infer_type_from_json_schema(sub_schema)
+ if inferred_type:
+ types.append(inferred_type)
+ if types:
+ if len(set(types)) == 1:
+ return types[0]
+ if len(set(types)) == 2 and "null" in types:
+ return [t for t in types if t != "null"][0]
+ if "string" in types:
+ return "string"
+ return types[0]
+
+ if "enum" in schema and isinstance(schema["enum"], list):
+ if not schema["enum"]:
+ return "string"
+ enum_types = set()
+ for value in schema["enum"]:
+ if value is None:
+ enum_types.add("null")
+ elif isinstance(value, bool):
+ enum_types.add("boolean")
+ elif isinstance(value, int):
+ enum_types.add("integer")
+ elif isinstance(value, float):
+ enum_types.add("number")
+ elif isinstance(value, str):
+ enum_types.add("string")
+ elif isinstance(value, list):
+ enum_types.add("array")
+ elif isinstance(value, dict):
+ enum_types.add("object")
+ if len(enum_types) == 1:
+ return enum_types.pop()
+ return "string"
+
+ if "allOf" in schema and isinstance(schema["allOf"], list):
+ schemas = schema["allOf"]
+ for sub_schema in schemas:
+ inferred_type = infer_type_from_json_schema(sub_schema)
+ if inferred_type and inferred_type != "string":
+ return inferred_type
+ return "string"
+
+ if "properties" in schema:
+ return "object"
+ if "items" in schema:
+ return "array"
+
+ return None
+
+
+def _convert_to_number(value: str) -> Any:
+ """Convert string to appropriate number type (int or float)."""
+ try:
+ if "." in value or "e" in value.lower():
+ return float(value)
+ else:
+ return int(value)
+ except (ValueError, AttributeError):
+ return value
+
+
+def parse_arguments(
+ json_value: str, arg_type: Optional[str] = None
+) -> Tuple[Any, bool]:
+ """Parse argument value with multiple fallback strategies."""
+ if not isinstance(json_value, str):
+ return json_value, True
+ try:
+ parsed_value = json.loads(json_value)
+ if arg_type == "number" and isinstance(parsed_value, str):
+ parsed_value = _convert_to_number(parsed_value)
+ return parsed_value, True
+ except (json.JSONDecodeError, ValueError):
+ pass
+
+ try:
+ wrapped = json.loads('{"tmp": "' + json_value + '"}')
+ parsed_value = json.loads(wrapped["tmp"])
+ if arg_type == "number" and isinstance(parsed_value, str):
+ parsed_value = _convert_to_number(parsed_value)
+ return parsed_value, True
+ except (json.JSONDecodeError, ValueError, KeyError):
+ pass
+
+ if arg_type == "string":
+ if (
+ len(json_value) >= 2
+ and json_value[0] == json_value[-1]
+ and json_value[0] in {'"', "'"}
+ ):
+ return json_value[1:-1], True
+ return json_value, True
+
+ try:
+ parsed_value = safe_literal_eval(json_value)
+ return parsed_value, True
+ except (ValueError, SyntaxError):
+ pass
+
+ try:
+ quoted_value = json.dumps(str(json_value))
+ return json.loads(quoted_value), True
+ except (json.JSONDecodeError, ValueError):
+ return json_value, False
diff --git a/python/infinilm/base_config.py b/python/infinilm/base_config.py
index 4ae7665c0..7b7affb9e 100644
--- a/python/infinilm/base_config.py
+++ b/python/infinilm/base_config.py
@@ -118,6 +118,8 @@ def __init__(self):
self.port = self.args.port
self.endpoint = self.args.endpoint
self.ignore_eos = self.args.ignore_eos
+ self.tool_call_parser = self.args.tool_call_parser
+ self.reasoning_parser = self.args.reasoning_parser
# PD separation (KV transfer)
self.kv_transfer_config = self.args.kv_transfer_config
@@ -416,6 +418,18 @@ def _add_common_args(self):
default=False,
help="Ignore EOS token and continue generation",
)
+ self.parser.add_argument(
+ "--tool-call-parser",
+ type=str,
+ default=None,
+ help="Tool-call parser to use (e.g., glm, llama31)",
+ )
+ self.parser.add_argument(
+ "--reasoning-parser",
+ type=str,
+ default=None,
+ help="Reasoning parser to use (e.g., glm45, deepseek-r1)",
+ )
# --- Multimodal parameters ---
self.parser.add_argument(
diff --git a/python/infinilm/llm/llm.py b/python/infinilm/llm/llm.py
index 59d5a1eca..7dfe84828 100644
--- a/python/infinilm/llm/llm.py
+++ b/python/infinilm/llm/llm.py
@@ -321,9 +321,12 @@ def apply_chat_template(
messages: List[dict],
add_generation_prompt: bool = True,
chat_template_kwargs: Optional[dict] = None,
+ tools: Optional[List[dict]] = None,
) -> str:
"""Apply chat template to messages."""
chat_template_kwargs = chat_template_kwargs or {}
+ if tools is not None:
+ chat_template_kwargs["tools"] = tools
return self.processor.apply_chat_template(
conversation=messages,
add_generation_prompt=add_generation_prompt,
@@ -784,6 +787,7 @@ def add_request(
request_id: Optional[str] = None,
# For server use
request_data: Optional[dict] = None,
+ chat_template_kwargs: Optional[dict] = None,
) -> InferenceRequest:
"""Add a request to the engine.
@@ -813,6 +817,8 @@ def add_request(
sampling_params: Sampling parameters.
request_id: Optional request ID.
request_data: Optional request data dict (for server use).
+ chat_template_kwargs: Optional extra keyword arguments forwarded to
+ ``apply_chat_template()`` (e.g. ``tools`` definitions).
Returns:
The created InferenceRequest object.
@@ -838,7 +844,9 @@ def add_request(
)
prompt = self.engine.apply_chat_template(
- messages, add_generation_prompt=add_generation_prompt
+ messages,
+ add_generation_prompt=add_generation_prompt,
+ chat_template_kwargs=chat_template_kwargs,
)
mm_inputs = resolve_multimodal_inputs(messages)
@@ -897,7 +905,7 @@ def add_chat_request(
request_id: Optional[str] = None,
request_data: Optional[dict] = None,
add_generation_prompt: bool = True,
- **kwargs,
+ chat_template_kwargs: Optional[dict] = None,
) -> InferenceRequest:
"""Add a chat request to the engine.
@@ -906,6 +914,9 @@ def add_chat_request(
sampling_params: Sampling parameters.
request_id: Optional request ID.
request_data: Optional request data dict.
+ add_generation_prompt: Whether to add a generation prompt.
+ chat_template_kwargs: Optional extra keyword arguments forwarded to
+ ``apply_chat_template()`` (e.g. ``tools`` definitions).
Returns:
The created InferenceRequest object.
@@ -918,6 +929,7 @@ def add_chat_request(
sampling_params=sampling_params,
request_id=request_id,
request_data=request_data,
+ chat_template_kwargs=chat_template_kwargs,
)
async def stream_request(
diff --git a/python/infinilm/processors/basic_llm_processor.py b/python/infinilm/processors/basic_llm_processor.py
index a6fbc33ac..c6c50d947 100644
--- a/python/infinilm/processors/basic_llm_processor.py
+++ b/python/infinilm/processors/basic_llm_processor.py
@@ -35,6 +35,42 @@ def __call__(self, prompt: str, return_tensors: str = None, **kwargs) -> dict:
# "pt" or "np" or "tf".
return self.tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
+ @staticmethod
+ def normalize_conversation(conversation):
+ """Normalize message content for chat-template rendering.
+
+ Chat templates render message content as a string, but chat/agent
+ clients commonly send text as a content-part list
+ (``[{"type": "text", "text": ...}, ...]``). Text-only lists are
+ joined into a single string here; content containing non-text parts
+ is rejected, since multimodal media is handled by the dedicated
+ multimodal processors instead.
+ """
+ normalized = []
+ for message in conversation:
+ content = message.get("content")
+ if isinstance(content, list):
+ if not all(
+ isinstance(item, dict)
+ and item.get("type") == "text"
+ and "text" in item
+ for item in content
+ ):
+ raise ValueError(
+ "Only text content parts are supported in message "
+ "content lists; media parts require a multimodal "
+ "model/processor."
+ )
+ normalized.append(
+ {
+ **message,
+ "content": "".join(item["text"] for item in content),
+ }
+ )
+ else:
+ normalized.append(message)
+ return normalized
+
@override
def apply_chat_template(
self,
@@ -43,23 +79,8 @@ def apply_chat_template(
tokenize: bool = True,
**kwargs,
):
- normalized_conversation = []
- for message in conversation:
- if isinstance(message["content"], list):
- assert len(message["content"]) == 1, (
- "Only one content item supported in list"
- )
- content_item = message["content"][0]
- assert "type" in content_item and "text" in content_item, (
- "Content dict must have 'type' and 'text' keys"
- )
- normalized_conversation.append(
- {"role": message["role"], "content": content_item["text"]}
- )
- else:
- normalized_conversation.append(message)
return self.tokenizer.apply_chat_template(
- conversation=normalized_conversation,
+ conversation=self.normalize_conversation(conversation),
add_generation_prompt=add_generation_prompt,
tokenize=tokenize,
**kwargs,
diff --git a/python/infinilm/server/inference_server.py b/python/infinilm/server/inference_server.py
index 462c31084..cbf992699 100644
--- a/python/infinilm/server/inference_server.py
+++ b/python/infinilm/server/inference_server.py
@@ -15,6 +15,16 @@
import uvicorn
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, StreamingResponse
+from infinilm.agents import AgentStreamParser, parse_full_response
+from infinilm.agents.anthropic import (
+ AnthropicMessagesRequest,
+ anthropic_error_body,
+ convert_anthropic_request,
+ convert_openai_sse_stream,
+ convert_openai_to_anthropic_response,
+)
+from infinilm.agents.message_adapter import adapt_messages, prepare_chat_template_kwargs
+from infinilm.agents.protocol import chunk_json, completion_json
from infinilm.base_config import BaseConfig
from infinilm.config import KVTransferConfig
from infinilm.llm import AsyncLLMEngine, FinishReason, SamplingParams
@@ -26,69 +36,6 @@
DEFAULT_REQUEST_TIMEOUT = 1000.0
-def chunk_json(
- id_, content=None, role=None, finish_reason=None, model: str = "unknown"
-):
- """Generate JSON chunk for streaming response."""
- delta = {}
- if content:
- delta["content"] = content
- if role:
- delta["role"] = role
- return {
- "id": id_,
- "object": "chat.completion.chunk",
- "created": int(time.time()),
- "model": model,
- "system_fingerprint": None,
- "choices": [
- {
- "index": 0,
- "text": content,
- "delta": delta,
- "logprobs": None,
- "finish_reason": finish_reason,
- }
- ],
- }
-
-
-def completion_json(
- id_,
- content,
- role="assistant",
- finish_reason="stop",
- model: str = "unknown",
- prompt_tokens: int = 0,
- completion_tokens: int = 0,
- total_tokens: int = 0,
-):
- """Generate JSON response for non-streaming completion."""
- return {
- "id": id_,
- "object": "chat.completion",
- "created": int(time.time()),
- "model": model,
- "system_fingerprint": None,
- "choices": [
- {
- "index": 0,
- "message": {
- "role": role,
- "content": content,
- },
- "logprobs": None,
- "finish_reason": finish_reason,
- }
- ],
- "usage": {
- "prompt_tokens": prompt_tokens,
- "completion_tokens": completion_tokens,
- "total_tokens": total_tokens,
- },
- }
-
-
class InferenceServer:
"""HTTP server for LLM inference."""
@@ -125,6 +72,8 @@ def __init__(
kv_transfer_config: Optional[KVTransferConfig] = None,
enable_prefix_caching: bool = True,
pre_transpose: bool = False,
+ tool_call_parser: Optional[str] = None,
+ reasoning_parser: Optional[str] = None,
):
"""Initialize inference server.
@@ -188,6 +137,8 @@ def __init__(
self.kv_transfer_config = kv_transfer_config
self.enable_prefix_caching = enable_prefix_caching
self.pre_transpose = pre_transpose
+ self.tool_call_parser = tool_call_parser
+ self.reasoning_parser = reasoning_parser
self.engine: AsyncLLMEngine = None
@@ -266,13 +217,11 @@ async def chat_completions(request: Request):
else:
data["messages"] = [{"role": "user", "content": data.get("prompt")}]
- # Normalize messages to handle multimodal content (list format)
data["messages"] = data.get("messages", [])
-
- stream = data.get("stream", False)
request_id = f"cmpl-{uuid.uuid4().hex}"
+ prepare_chat_template_kwargs(data)
- if stream:
+ if data.get("stream", False):
return StreamingResponse(
self._stream_chat(request_id, data, request),
media_type="text/event-stream",
@@ -283,6 +232,38 @@ async def chat_completions(request: Request):
return response
return JSONResponse(content=response)
+ # Anthropic-compatible Messages API endpoint.
+ @app.post("/v1/messages")
+ async def anthropic_messages(request: Request):
+ try:
+ anthropic_req = AnthropicMessagesRequest(**await request.json())
+ openai_data = convert_anthropic_request(anthropic_req)
+ except Exception as e:
+ logger.error(f"Failed to parse Anthropic request: {e}")
+ return JSONResponse(
+ content=anthropic_error_body(str(e)), status_code=400
+ )
+
+ request_id = f"msg_{uuid.uuid4().hex}"
+ prepare_chat_template_kwargs(openai_data)
+
+ if openai_data.get("stream", False):
+ return StreamingResponse(
+ self._anthropic_stream(request_id, openai_data, request),
+ media_type="text/event-stream",
+ )
+ response = await self._chat(request_id, openai_data, request)
+ if isinstance(response, JSONResponse):
+ return response
+ return JSONResponse(
+ content=convert_openai_to_anthropic_response(response, self.model_id)
+ )
+
+ @app.head("/api/hello")
+ @app.get("/api/hello")
+ async def api_hello():
+ return {"status": "ok"}
+
@app.get("/health")
async def health():
# Expose engine health so babysitter/registry can treat backend as unhealthy.
@@ -316,39 +297,6 @@ async def list_models():
async def list_models_legacy():
return _models_payload()
- def _normalize_messages(self, messages: list) -> list:
- """Normalize messages to handle multimodal content (list format).
-
- Converts content from list format [{"type": "text", "text": "..."}]
- to string format for chat template compatibility.
- """
- normalized = []
- for msg in messages:
- if not isinstance(msg, dict):
- normalized.append(msg)
- continue
-
- content = msg.get("content")
- if isinstance(content, list):
- # Extract text from multimodal content list
- text_parts = []
- for part in content:
- if isinstance(part, dict):
- if part.get("type") == "text" and "text" in part:
- text_parts.append(part["text"])
- elif isinstance(part, str):
- text_parts.append(part)
- elif isinstance(part, str):
- text_parts.append(part)
- # Join all text parts
- normalized_msg = msg.copy()
- normalized_msg["content"] = "".join(text_parts) if text_parts else ""
- normalized.append(normalized_msg)
- else:
- normalized.append(msg)
-
- return normalized
-
def _build_sampling_params(self, data: dict) -> SamplingParams:
"""Build SamplingParams from request data."""
# Support both:
@@ -391,7 +339,15 @@ async def _stream_chat(self, request_id: str, data: dict, http_request: Request)
_abort_reason = FinishReason.CANCELED
try:
- messages = data.get("messages", [])
+ # The stream parser is stateful; a dedicated instance per request
+ # keeps concurrent streams from clobbering each other's buffers.
+ agent_parser = AgentStreamParser(
+ tool_call_parser=self.tool_call_parser,
+ reasoning_parser=self.reasoning_parser,
+ tools=data.get("tools") or [],
+ )
+
+ messages = adapt_messages(data.get("messages", []), self.tool_call_parser)
sampling_params = self._build_sampling_params(data)
req = self.engine.add_chat_request(
@@ -439,24 +395,34 @@ async def _stream_chat(self, request_id: str, data: dict, http_request: Request)
)
if not is_eos_token and token_output.token_text:
- # Send token
- chunk = json.dumps(
- chunk_json(
- request_id,
- content=token_output.token_text,
- model=self.model_id,
- ),
- ensure_ascii=False,
- )
- yield f"data: {chunk}\n\n"
+ # All parsing and protocol formatting lives in the agents
+ # package; the server only relays the rendered SSE lines.
+ for event in agent_parser.delta_events(
+ token_output.token_text, request_id, self.model_id
+ ):
+ yield event
if token_output.finished:
+ # Flush any remaining buffered tool-call arguments before
+ # emitting the final finish chunk.
+ for event in agent_parser.flush_events(request_id, self.model_id):
+ yield event
+
finish_reason = self._convert_finish_reason(
token_output.finish_reason
)
+ if agent_parser.has_tool_calls:
+ finish_reason = "tool_calls"
chunk = json.dumps(
chunk_json(
- request_id, finish_reason=finish_reason, model=self.model_id
+ request_id,
+ finish_reason=finish_reason,
+ model=self.model_id,
+ usage={
+ "prompt_tokens": req.get_prompt_length(),
+ "completion_tokens": req.get_num_generated_tokens(),
+ "total_tokens": req.get_total_length(),
+ },
),
ensure_ascii=False,
)
@@ -496,7 +462,7 @@ async def _chat(self, request_id: str, data: dict, http_request: Request):
_abort_reason = FinishReason.CANCELED
try:
- messages = data.get("messages", [])
+ messages = adapt_messages(data.get("messages", []), self.tool_call_parser)
sampling_params = self._build_sampling_params(data)
req = self.engine.add_chat_request(
@@ -537,17 +503,31 @@ async def _chat(self, request_id: str, data: dict, http_request: Request):
break
output_text = output_text.strip()
+
+ # One-shot parse of the complete response into reasoning /
+ # content / tool_calls (all parsing lives in the agents package).
+ reasoning_content, normal_text, tool_calls = parse_full_response(
+ output_text,
+ tool_call_parser=self.tool_call_parser,
+ reasoning_parser=self.reasoning_parser,
+ tools=data.get("tools") or [],
+ )
+
finish_reason = self._convert_finish_reason(req.finish_reason)
+ if tool_calls:
+ finish_reason = "tool_calls"
response = completion_json(
request_id,
- content=output_text,
+ content=normal_text,
role="assistant",
finish_reason=finish_reason or "stop",
model=self.model_id,
prompt_tokens=req.get_prompt_length(),
completion_tokens=req.get_num_generated_tokens(),
total_tokens=req.get_total_length(),
+ reasoning_content=reasoning_content,
+ tool_calls=tool_calls,
)
return response
@@ -575,6 +555,16 @@ def _convert_finish_reason(self, reason: FinishReason) -> str:
return reason.value
+ # ---------- Anthropic stream conversion ----------
+
+ def _anthropic_stream(self, request_id: str, data: dict, http_request: Request):
+ """Consume the OpenAI-format stream and emit Anthropic-format SSE events."""
+ return convert_openai_sse_stream(
+ self._stream_chat(request_id, data, http_request),
+ message_id=f"msg_{uuid.uuid4().hex}",
+ model=self.model_id,
+ )
+
def setup_logging(log_level: str = "INFO"):
"""Configure logging system with proper formatting and handlers."""
@@ -667,6 +657,8 @@ def main():
kv_transfer_config=kv_transfer_config,
enable_prefix_caching=cfg.enable_prefix_caching,
pre_transpose=cfg.pre_transpose,
+ tool_call_parser=cfg.tool_call_parser,
+ reasoning_parser=cfg.reasoning_parser,
)
server.start()
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 000000000..6d48395af
--- /dev/null
+++ b/test/__init__.py
@@ -0,0 +1,13 @@
+# Package marker for the repository test suite -- please do not delete.
+#
+# This file makes ``test/`` a regular Python package so that dotted-path
+# unittest invocations work from the repository root:
+#
+# python -m unittest test.agents.test_agents
+#
+# Without it, ``import test`` resolves to the *standard library* ``test``
+# package instead of this directory, and the command above fails with
+# "No module named 'test.agents'".
+#
+# Note: this intentionally shadows the stdlib ``test`` package while running
+# the project's own tests; nothing in the test suite imports the stdlib one.
diff --git a/test/agents/__init__.py b/test/agents/__init__.py
new file mode 100644
index 000000000..12861ecdb
--- /dev/null
+++ b/test/agents/__init__.py
@@ -0,0 +1,7 @@
+# Package marker for the agent-support test suite -- please do not delete.
+#
+# Together with ``test/__init__.py`` this makes the test modules importable
+# by dotted path so both entry points work from the repository root:
+#
+# python -m unittest test.agents.test_agents
+# python -m unittest discover -s test/agents
diff --git a/test/agents/test_agents.py b/test/agents/test_agents.py
new file mode 100644
index 000000000..2eea1d9ac
--- /dev/null
+++ b/test/agents/test_agents.py
@@ -0,0 +1,1682 @@
+"""
+Basic unit tests for agent support modules.
+These tests do not require GPU and can run in a pure Python environment.
+"""
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+# Ensure infinilm is importable when running tests directly. The Python
+# sources live under /python.
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+PYTHON_ROOT = PROJECT_ROOT / "python"
+for _path in (str(PYTHON_ROOT), str(PROJECT_ROOT)):
+ if _path not in sys.path:
+ sys.path.insert(0, _path)
+
+try:
+ from infinilm.agents import (
+ Function,
+ FunctionCallParser,
+ ReasoningParser,
+ StreamingParseResult,
+ Tool,
+ ToolCallItem,
+ )
+ from infinilm.agents.utils import parse_arguments
+except Exception:
+ # ``infinilm/__init__.py`` eagerly loads the native engine stack, which
+ # the pure-Python agents package does not need. Register a minimal
+ # namespace stub so ``infinilm.agents`` can be imported standalone in
+ # environments without the compiled extension (CPU-only test setups).
+ import types
+
+ sys.modules.pop("infinilm", None)
+ _infinilm_stub = types.ModuleType("infinilm")
+ _infinilm_stub.__path__ = [str(PYTHON_ROOT / "infinilm")]
+ sys.modules["infinilm"] = _infinilm_stub
+
+ from infinilm.agents import (
+ Function,
+ FunctionCallParser,
+ ReasoningParser,
+ StreamingParseResult,
+ Tool,
+ ToolCallItem,
+ )
+ from infinilm.agents.utils import parse_arguments
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_weather_tools(backend="llama31"):
+ """Return a list of Tool objects for get_weather / get_time tools."""
+ weather_tool = Tool(
+ type="function",
+ function=Function(
+ name="get_weather",
+ description="Get weather for a city",
+ parameters={
+ "type": "object",
+ "properties": {
+ "city": {"type": "string", "description": "City name"},
+ },
+ "required": ["city"],
+ },
+ ),
+ )
+ time_tool = Tool(
+ type="function",
+ function=Function(
+ name="get_time",
+ description="Get current time",
+ parameters={"type": "object", "properties": {}},
+ ),
+ )
+ return [weather_tool, time_tool]
+
+
+def _make_weather_dict_tools():
+ """Return weather/time tools as plain dicts (like from HTTP JSON)."""
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get weather for a city",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "city": {"type": "string", "description": "City name"},
+ },
+ "required": ["city"],
+ },
+ },
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "get_time",
+ "description": "Get current time",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ },
+ ]
+
+
+# ===========================================================================
+# ReasoningParser
+# ===========================================================================
+
+
+class TestReasoningParser(unittest.TestCase):
+ """Tests for ReasoningParser and BaseReasoningFormatDetector."""
+
+ # ---- GLM45 (keep existing) ------------------------------------------------
+
+ def test_glm45_detector_non_streaming(self):
+ parser = ReasoningParser(reasoning_parser_name="glm45")
+ text = "thinkI need to think step by step./thinkThe answer is 42."
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 2)
+ self.assertEqual(results[0].reasoning_content, "I need to think step by step.")
+ self.assertEqual(results[0].complete, True)
+ self.assertEqual(results[1].normal_text, "The answer is 42.")
+
+ def test_glm45_detector_streaming(self):
+ parser = ReasoningParser(reasoning_parser_name="glm45")
+ # Simulate streaming: prefill already started reasoning.
+ # The start_token is not in the stream because prefill included it.
+ # The base detector treats content as reasoning until end_token appears.
+ deltas = ["Thinking...", " done", "/thinkAnswer: 42"]
+ current = ""
+ normal_parts = []
+ reasoning_parts = []
+ for d in deltas:
+ prev = current
+ current += d
+ res = parser.extract_reasoning_content_streaming(prev, current, d)
+ if res.reasoning_content:
+ reasoning_parts.append(res.reasoning_content)
+ if res.normal_text:
+ normal_parts.append(res.normal_text)
+
+ # Verify that reasoning content is emitted incrementally
+ self.assertIn("Thinking... done", "".join(reasoning_parts))
+ # After end_token, normal text is emitted
+ self.assertIn("Answer: 42", "".join(normal_parts))
+
+ # ---- Think tag detector ---------------------------------------------------
+
+ def test_think_tag_detector_non_streaming(self):
+ """ReasoningParser(name='think') extracts content between tags."""
+ parser = ReasoningParser(reasoning_parser_name="think")
+ text = (
+ "Let me reason about this carefully.The answer is 42."
+ )
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 2)
+ self.assertEqual(
+ results[0].reasoning_content, "Let me reason about this carefully."
+ )
+ self.assertEqual(results[0].complete, True)
+ self.assertEqual(results[1].normal_text, "The answer is 42.")
+
+ def test_think_tag_detector_streaming(self):
+ """Stream tokens through ThinkTagDetector and verify accumulation."""
+ parser = ReasoningParser(reasoning_parser_name="think")
+ text = "Step 1: Analyse.Final answer: yes."
+ current = ""
+ reasoning_parts = []
+ normal_parts = []
+ # Feed character by character to stress-test streaming
+ for ch in text:
+ prev = current
+ current += ch
+ res = parser.extract_reasoning_content_streaming(prev, current, ch)
+ if res.reasoning_content:
+ reasoning_parts.append(res.reasoning_content)
+ if res.normal_text:
+ normal_parts.append(res.normal_text)
+
+ self.assertEqual("".join(reasoning_parts), "Step 1: Analyse.")
+ self.assertEqual("".join(normal_parts), "Final answer: yes.")
+
+ def test_think_tag_detector_streaming_word_chunks(self):
+ """Stream larger word-level chunks through ThinkTagDetector."""
+ parser = ReasoningParser(reasoning_parser_name="think")
+ # Use a single ... pair split across chunks.
+ chunks = ["First ", "thought.done"]
+ current = ""
+ reasoning_parts = []
+ normal_parts = []
+ for chunk in chunks:
+ prev = current
+ current += chunk
+ res = parser.extract_reasoning_content_streaming(prev, current, chunk)
+ if res.reasoning_content:
+ reasoning_parts.append(res.reasoning_content)
+ if res.normal_text:
+ normal_parts.append(res.normal_text)
+
+ full_reasoning = "".join(reasoning_parts)
+ self.assertIn("First thought.", full_reasoning)
+ self.assertEqual("".join(normal_parts), "done")
+
+ # ---- No-op parser ----------------------------------------------------------
+
+ def test_no_op_parser(self):
+ parser = ReasoningParser(reasoning_parser_name=None)
+ res = parser.extract_reasoning_content_streaming("", "hello", "hello")
+ self.assertEqual(res.normal_text, "hello")
+ self.assertEqual(res.reasoning_content, "")
+
+ # ---- Think tag without think tags -------------------------------------------
+
+ def test_think_tag_no_think(self):
+ """Text without tags should all be returned as normal_text."""
+ parser = ReasoningParser(reasoning_parser_name="think")
+ text = "Hello, world! No reasoning here, just a plain response."
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results[0].reasoning_content, "")
+ self.assertEqual(results[0].normal_text, text)
+ self.assertEqual(results[0].complete, True)
+
+ def test_think_tag_partial_no_close(self):
+ """Text with opening tag but no closing tag."""
+ parser = ReasoningParser(reasoning_parser_name="think")
+ text = "Partial reasoning here"
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results[0].reasoning_content, "Partial reasoning here")
+ self.assertEqual(results[0].normal_text, "")
+ self.assertEqual(results[0].complete, False)
+
+ # ---- DeepSeek-R1 / QwQ ( tag) ---------------------------------------
+
+ def test_deepseek_r1_detector_non_streaming(self):
+ """deepseek-r1 uses the short tag, not ."""
+ parser = ReasoningParser(reasoning_parser_name="deepseek-r1")
+ text = "hidden reasoninganswer"
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 2)
+ self.assertEqual(results[0].reasoning_content, "hidden reasoning")
+ self.assertEqual(results[0].complete, True)
+ self.assertEqual(results[1].normal_text, "answer")
+
+ def test_qwq_detector_non_streaming(self):
+ """qwq is an alias for the same tag format."""
+ parser = ReasoningParser(reasoning_parser_name="qwq")
+ text = "Let me think.42"
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 2)
+ self.assertEqual(results[0].reasoning_content, "Let me think.")
+ self.assertEqual(results[1].normal_text, "42")
+
+ def test_deepseek_r1_detector_streaming(self):
+ """Stream a block character by character."""
+ parser = ReasoningParser(reasoning_parser_name="deepseek-r1")
+ text = "Step 1: analyse the problem.The answer is 42."
+ current = ""
+ reasoning_parts = []
+ normal_parts = []
+ for ch in text:
+ prev = current
+ current += ch
+ res = parser.extract_reasoning_content_streaming(prev, current, ch)
+ if res.reasoning_content:
+ reasoning_parts.append(res.reasoning_content)
+ if res.normal_text:
+ normal_parts.append(res.normal_text)
+
+ self.assertEqual("".join(reasoning_parts), "Step 1: analyse the problem.")
+ self.assertEqual("".join(normal_parts), "The answer is 42.")
+
+ def test_deepseek_r1_detector_chunked_streaming(self):
+ """Stream a block in chunks split across tag boundaries."""
+ parser = ReasoningParser(reasoning_parser_name="deepseek-r1")
+ chunks = ["deep ", "thoughtsfinal answer"]
+ current = ""
+ reasoning_parts = []
+ normal_parts = []
+ for chunk in chunks:
+ prev = current
+ current += chunk
+ res = parser.extract_reasoning_content_streaming(prev, current, chunk)
+ if res.reasoning_content:
+ reasoning_parts.append(res.reasoning_content)
+ if res.normal_text:
+ normal_parts.append(res.normal_text)
+
+ self.assertEqual("".join(reasoning_parts), "deep thoughts")
+ self.assertEqual("".join(normal_parts), "final answer")
+
+ def test_deepseek_alias_does_not_match_thinking_tag(self):
+ """deepseek-r1 must not silently fall back to parsing."""
+ parser = ReasoningParser(reasoning_parser_name="deepseek-r1")
+ text = "not the r1 formatanswer"
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results[0].normal_text, text)
+ self.assertEqual(results[0].reasoning_content, "")
+
+ # ---- Edge cases ------------------------------------------------------------
+
+ def test_unknown_reasoning_parser_name(self):
+ """Passing an unknown parser name produces empty detectors (same as no-op)."""
+ parser = ReasoningParser(reasoning_parser_name="nonexistent")
+ text = "testanswer"
+ results = parser.extract_reasoning_content(text)
+ self.assertEqual(len(results), 1)
+ self.assertEqual(results[0].normal_text, text)
+ self.assertEqual(results[0].reasoning_content, "")
+
+
+# ===========================================================================
+# ToolCallParser
+# ===========================================================================
+
+
+class TestToolCallParser(unittest.TestCase):
+ """Tests for FunctionCallParser with various backends."""
+
+ # ---- Registration (keep existing) ------------------------------------------
+
+ def test_llama_registration(self):
+ parser = FunctionCallParser(tool_call_parser="llama31")
+ self.assertIsNotNone(parser.detector)
+
+ def test_glm_registration(self):
+ parser = FunctionCallParser(tool_call_parser="glm")
+ self.assertIsNotNone(parser.detector)
+
+ def test_non_stream_simple(self):
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=[])
+ text = "Hello world"
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal, text)
+ self.assertEqual(calls, [])
+
+ # ---- Llama 3.1 non-stream with tools ---------------------------------------
+
+ def test_llama31_non_stream_with_tools(self):
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ text = (
+ 'Hello<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+ )
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal.strip(), "Hello")
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "Beijing"})
+
+ def test_llama31_non_stream_with_tools_no_prefix(self):
+ """Tool call starts immediately without leading text."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ text = '<|python_tag|>{"name":"get_time", "arguments":{}}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal.strip(), "")
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_time")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {})
+
+ def test_llama31_non_stream_no_tool_match(self):
+ """Model calls a tool not in the provided list."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ text = '<|python_tag|>{"name":"nonexistent_fn", "arguments":{"x":1}}'
+ normal, calls = parser.parse_non_stream(text)
+ # Undefined tools are filtered out by the detector.
+ self.assertEqual(len(calls), 0)
+
+ # ---- Llama 3.1 streaming with tools ----------------------------------------
+
+ def test_llama31_stream_with_tools(self):
+ """Stream a complete tool call in chunks; verify name then params."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ call_text = (
+ '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+ )
+
+ # Feed in two chunks to simulate streaming.
+ # Split right after the bot token so the first chunk never contains
+ # partial JSON keys / values that could confuse the incremental parser.
+ bot_len = len("<|python_tag|>")
+ result1 = parser.parse_streaming_increment("", call_text[:bot_len], tools)
+ result2 = parser.parse_streaming_increment("", call_text[bot_len:], tools)
+ _, end_calls = parser.parse_stream_end()
+
+ all_calls = result1.calls + result2.calls + end_calls
+ all_names = [c.name for c in all_calls if c.name]
+ self.assertIn("get_weather", all_names)
+ # All streamed argument fragments must reassemble into the full JSON.
+ streamed_args = "".join(c.parameters for c in all_calls)
+ self.assertEqual(json.loads(streamed_args), {"city": "Beijing"})
+
+ def test_llama31_stream_with_tools_single_chunk(self):
+ """Stream entire JSON block at once; arguments must not be lost."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ call_text = '<|python_tag|>{"name":"get_time", "arguments":{}}'
+ result = parser.parse_streaming_increment("", call_text, tools)
+ _, end_calls = parser.parse_stream_end()
+
+ all_calls = result.calls + end_calls
+ names = [c.name for c in all_calls if c.name]
+ self.assertIn("get_time", names)
+ streamed_args = "".join(c.parameters for c in all_calls)
+ self.assertEqual(json.loads(streamed_args), {})
+
+ def test_llama31_stream_token_by_token(self):
+ """Stream a tool call one character at a time; params must survive."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ call_text = (
+ '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+ )
+
+ names = []
+ arg_parts = []
+ for ch in call_text:
+ res = parser.parse_streaming_increment("", ch, tools)
+ for call in res.calls:
+ if call.name:
+ names.append(call.name)
+ if call.parameters:
+ arg_parts.append(call.parameters)
+ _, end_calls = parser.parse_stream_end()
+ for call in end_calls:
+ if call.name:
+ names.append(call.name)
+ if call.parameters:
+ arg_parts.append(call.parameters)
+
+ self.assertIn("get_weather", names)
+ self.assertEqual(json.loads("".join(arg_parts)), {"city": "Beijing"})
+
+ def test_llama31_stream_end_flushes_buffered_args(self):
+ """parse_stream_end() must flush buffered arguments.
+
+ Regression test for the reviewer repro: a parser constructed without
+ bound tools used to return an empty result from parse_stream_end(),
+ leaving the streamed arguments stuck in the detector buffer.
+ """
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31")
+ call_text = (
+ '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+ )
+ # Split inside the arguments object so part of it stays buffered.
+ split = len('<|python_tag|>{"name":"get_weather", "arguments":{"ci')
+ increment_results = [
+ parser.parse_streaming_increment("", call_text[:split], tools),
+ parser.parse_streaming_increment("", call_text[split:], tools),
+ ]
+ normal, calls = parser.parse_stream_end(tools)
+
+ all_calls = increment_results[0].calls + increment_results[1].calls + calls
+ self.assertIn("get_weather", [c.name for c in all_calls if c.name])
+ streamed_args = "".join(c.parameters for c in all_calls)
+ self.assertEqual(json.loads(streamed_args), {"city": "Beijing"})
+ self.assertEqual(normal, "")
+
+ def test_llama31_plain_json_response_not_swallowed_streaming(self):
+ """A JSON answer without a tool name must not be held back forever."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ text = '{"answer": 42}'
+ parts = []
+ for ch in text:
+ res = parser.parse_streaming_increment("", ch, tools)
+ if res.normal_text:
+ parts.append(res.normal_text)
+ normal, calls = parser.parse_stream_end()
+ if normal:
+ parts.append(normal)
+ self.assertEqual("".join(parts), text)
+ self.assertEqual(calls, [])
+
+ def test_llama31_plain_json_response_not_swallowed_non_stream(self):
+ """Non-streaming parse also passes plain JSON answers through."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ text = '{"answer": 42}'
+ normal, calls = parser.parse_non_stream(text, tools=tools)
+ self.assertEqual(normal, text)
+ self.assertEqual(calls, [])
+
+ def test_llama31_stream_multiple_tools(self):
+ """Parse multiple tool calls in separate stream chunks."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ chunk1 = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}};'
+ chunk2 = '<|python_tag|>{"name":"get_time", "arguments":{}}'
+
+ result1 = parser.parse_streaming_increment("", chunk1, tools)
+ # Reset before the second chunk so state from the first call does not leak.
+ parser.detector.clear()
+ result2 = parser.parse_streaming_increment("", chunk2, tools)
+ all_names = []
+ all_params = []
+ for r in (result1, result2):
+ for c in r.calls:
+ if c.name:
+ all_names.append(c.name)
+ if c.parameters:
+ all_params.append(c.parameters)
+
+ self.assertIn("get_weather", all_names)
+ self.assertIn("get_time", all_names)
+
+ # ---- GLM non-stream with tools ---------------------------------------------
+
+ def test_glm_non_stream_with_tools(self):
+ """Parse GLM-style format."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="glm", tools=tools)
+ text = (
+ "Let me check.get_weather\n"
+ "city\nBeijing\n"
+ ""
+ )
+ normal, calls = parser.parse_non_stream(text)
+ self.assertIn("Let me check.", normal)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "Beijing"})
+
+ def test_glm_non_stream_multiple_tools(self):
+ """Parse multiple GLM tool calls in one text."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="glm", tools=tools)
+ text = (
+ "get_weather\n"
+ "city\nBeijing\n"
+ "\n"
+ "get_time\n"
+ ""
+ )
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(len(calls), 2)
+ self.assertEqual(calls[0].name, "get_weather")
+ self.assertEqual(calls[1].name, "get_time")
+ params0 = json.loads(calls[0].parameters)
+ self.assertEqual(params0, {"city": "Beijing"})
+
+ # ---- GLM streaming with tools -----------------------------------------------
+
+ def test_glm_stream_with_tools(self):
+ """Stream GLM tool call token by token."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="glm", tools=tools)
+ call_text = (
+ "some_prefixget_weather\n"
+ "city\nBeijing\n"
+ "tail"
+ )
+ result = parser.parse_streaming_increment("", call_text, tools)
+ # Should produce at least one ToolCallItem with name get_weather
+ names = [c.name for c in result.calls if c.name]
+ self.assertIn("get_weather", names)
+
+ def test_glm_detector_clear_resets_all_state(self):
+ """clear() must reset GLM streaming state so the parser is reusable."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="glm", tools=tools)
+ # Leave the detector mid-tool-call with dirty streaming state.
+ parser.parse_streaming_increment(
+ "", "get_weather\ncity"
+ )
+ parser.detector.clear()
+ # A fresh tool call must parse cleanly after the reset.
+ result = parser.parse_streaming_increment(
+ "", "get_time\n"
+ )
+ _, end_calls = parser.parse_stream_end()
+ all_calls = result.calls + end_calls
+ self.assertIn("get_time", [c.name for c in all_calls if c.name])
+ streamed_args = "".join(c.parameters for c in all_calls)
+ self.assertEqual(json.loads(streamed_args), {})
+
+ # ---- has_tool_call ---------------------------------------------------------
+
+ def test_has_tool_call(self):
+ """has_tool_call detects bot_token in text."""
+ # Llama
+ parser = FunctionCallParser(
+ tool_call_parser="llama31", tools=_make_weather_tools()
+ )
+ self.assertFalse(parser.has_tool_call("plain text"))
+ self.assertTrue(parser.has_tool_call("prefix<|python_tag|>json"))
+ self.assertTrue(parser.has_tool_call('{"name":"f"}'))
+
+ # GLM
+ parser2 = FunctionCallParser(
+ tool_call_parser="glm", tools=_make_weather_tools()
+ )
+ self.assertFalse(parser2.has_tool_call("plain text"))
+ self.assertTrue(
+ parser2.has_tool_call("before\nfn\nafter")
+ )
+
+ def test_has_tool_call_no_tools(self):
+ """has_tool_call returns False when no tools are configured."""
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=[])
+ self.assertFalse(parser.has_tool_call("<|python_tag|>json"))
+ self.assertFalse(parser.has_tool_call("plain"))
+
+ parser2 = FunctionCallParser(tool_call_parser="glm", tools=None)
+ self.assertFalse(parser2.has_tool_call("fn"))
+
+ # ---- parse_stream_end -------------------------------------------------------
+
+ def test_parse_stream_end(self):
+ """parse_stream_end flushes buffered state."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ normal, calls = parser.parse_stream_end()
+ self.assertEqual(normal, "")
+ self.assertEqual(calls, [])
+
+ def test_parse_stream_end_no_tools(self):
+ """parse_stream_end returns empty when no tools."""
+ parser = FunctionCallParser(tool_call_parser="llama31")
+ normal, calls = parser.parse_stream_end()
+ self.assertEqual(normal, "")
+ self.assertEqual(calls, [])
+
+ # ---- tools dict conversion --------------------------------------------------
+
+ def test_tools_dict_conversion(self):
+ """Passing plain dict tools works the same as Tool objects."""
+ dict_tools = _make_weather_dict_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=dict_tools)
+ text = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Shanghai"}}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "Shanghai"})
+
+ def test_tools_dict_conversion_glm(self):
+ """GLM parser also accepts plain dict tools."""
+ dict_tools = _make_weather_dict_tools()
+ parser = FunctionCallParser(tool_call_parser="glm", tools=dict_tools)
+ text = (
+ "get_weather\n"
+ "city\nShanghai\n"
+ ""
+ )
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "Shanghai"})
+
+ def test_tools_override_in_parse_call(self):
+ """tools passed at parse time override constructor tools."""
+ default_tools = _make_weather_tools()[0:1] # only get_weather
+ full_tools = _make_weather_tools() # get_weather + get_time
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=default_tools)
+ text = '<|python_tag|>{"name":"get_time", "arguments":{}}'
+ normal, calls = parser.parse_non_stream(text, tools=full_tools)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_time")
+
+
+# ===========================================================================
+# Glm4Chat0414Detector (GLM-4-9B-0414 metadata format)
+# ===========================================================================
+
+
+class TestGlm4Chat0414Parser(unittest.TestCase):
+ """Tests for the GLM-4-9B-0414 metadata-style tool call format:
+
+ function_name
+ {"arg": "value"}
+ """
+
+ def test_registration(self):
+ for name in ("glm4", "glm49b", "glm4-9b-0414", "glm-4-9b-0414"):
+ parser = FunctionCallParser(tool_call_parser=name)
+ self.assertIsNotNone(parser.detector)
+
+ def test_non_stream_simple_call(self):
+ """Exact shape observed from GLM-4-9B-0414 on the server."""
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = 'get_weather\n{"city": "北京"}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal, "")
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ self.assertEqual(json.loads(calls[0].parameters), {"city": "北京"})
+
+ def test_non_stream_prefix_text(self):
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = 'I will check the weather.\nget_weather\n{"city": "Shanghai"}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal, "I will check the weather.")
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(json.loads(calls[0].parameters), {"city": "Shanghai"})
+
+ def test_non_stream_plain_text_untouched(self):
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = "Just a normal answer with no tool call."
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal, text)
+ self.assertEqual(calls, [])
+
+ def test_non_stream_plain_json_answer_not_parsed_as_call(self):
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = 'The result is:\n{"answer": 42}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(calls, [])
+ self.assertIn('{"answer": 42}', normal)
+
+ def test_non_stream_unknown_tool_name_kept_as_text(self):
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = 'nonexistent_fn\n{"x": 1}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(calls, [])
+ self.assertIn("nonexistent_fn", normal)
+
+ def test_non_stream_multiple_calls_with_assistant_marker(self):
+ """Parallel calls are separated by <|assistant|> markers."""
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ marker = "<|" + "assistant|>"
+ text = 'get_weather\n{"city": "Beijing"}' + marker + "get_time\n{}"
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal, "")
+ self.assertEqual([c.name for c in calls], ["get_weather", "get_time"])
+ self.assertEqual(json.loads(calls[0].parameters), {"city": "Beijing"})
+ self.assertEqual(json.loads(calls[1].parameters), {})
+
+ def test_stream_char_by_char(self):
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = 'get_weather\n{"city": "北京"}'
+ names = []
+ arg_parts = []
+ for ch in text:
+ res = parser.parse_streaming_increment("", ch)
+ for call in res.calls:
+ if call.name:
+ names.append(call.name)
+ if call.parameters:
+ arg_parts.append(call.parameters)
+ normal, end_calls = parser.parse_stream_end()
+ for call in end_calls:
+ if call.name:
+ names.append(call.name)
+ if call.parameters:
+ arg_parts.append(call.parameters)
+
+ self.assertEqual(names, ["get_weather"])
+ self.assertEqual(json.loads("".join(arg_parts)), {"city": "北京"})
+ self.assertEqual(normal, "")
+
+ def test_stream_plain_text_not_held_back(self):
+ """Text that cannot become a tool call must stream out immediately."""
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ parts = []
+ for chunk in ["Hello ", "world, ", "no tools ", "here."]:
+ res = parser.parse_streaming_increment("", chunk)
+ if res.normal_text:
+ parts.append(res.normal_text)
+ self.assertEqual("".join(parts), "Hello world, no tools here.")
+
+ def test_stream_name_held_until_decidable(self):
+ """A line matching a tool-name prefix is held back, then resolves."""
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ res1 = parser.parse_streaming_increment("", "get_wea")
+ self.assertEqual(res1.normal_text, "")
+ self.assertEqual(res1.calls, [])
+ res2 = parser.parse_streaming_increment("", 'ther\n{"city": "Beijing"}')
+ self.assertEqual([c.name for c in res2.calls if c.name], ["get_weather"])
+ args = "".join(c.parameters for c in res2.calls)
+ self.assertEqual(json.loads(args), {"city": "Beijing"})
+
+ def test_stream_held_line_released_when_not_a_tool(self):
+ """A held line that turns out not to be a tool call is released."""
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ res1 = parser.parse_streaming_increment("", "get_we")
+ self.assertEqual(res1.normal_text, "")
+ # The line completes to something that is not a tool name.
+ res2 = parser.parse_streaming_increment("", "lcome to Beijing!")
+ self.assertIn("get_welcome to Beijing!", res2.normal_text)
+
+ def test_stream_multiple_calls_with_split_marker(self):
+ """Parallel calls whose <|assistant|> separator arrives split."""
+ marker = "<|" + "assistant|>"
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ text = 'get_weather\n{"city": "北京"}' + marker + "get_time\n{}"
+ names = []
+ arg_parts = []
+ # Feed in fixed awkward chunks that split inside the marker.
+ chunks = [text[:15], text[15:27], text[27:33], text[33:41], text[41:]]
+ fed = ""
+ for chunk in chunks:
+ self.assertTrue(text.startswith(fed + chunk))
+ fed += chunk
+ res = parser.parse_streaming_increment("", chunk)
+ for call in res.calls:
+ if call.name:
+ names.append(call.name)
+ if call.parameters:
+ arg_parts.append(call.parameters)
+ self.assertEqual(fed, text)
+ normal, end_calls = parser.parse_stream_end()
+ for call in end_calls:
+ if call.name:
+ names.append(call.name)
+ if call.parameters:
+ arg_parts.append(call.parameters)
+
+ self.assertEqual(names, ["get_weather", "get_time"])
+ self.assertNotIn(marker, normal)
+ self.assertNotIn("<|", normal)
+
+ def test_stream_marker_fragment_in_plain_text_not_duplicated(self):
+ """A partial marker inside plain text must not leak or duplicate."""
+ parser = FunctionCallParser(
+ tool_call_parser="glm4-9b-0414", tools=_make_weather_dict_tools()
+ )
+ marker = "<|" + "assistant|>"
+ text = "第一段" + marker + "第二段"
+ parts = []
+ for i in range(len(text)):
+ res = parser.parse_streaming_increment("", text[i])
+ if res.normal_text:
+ parts.append(res.normal_text)
+ normal, _ = parser.parse_stream_end()
+ if normal:
+ parts.append(normal)
+ combined = "".join(parts)
+ # Both text parts survive exactly once; the separator is dropped.
+ self.assertEqual(combined.count("第一段"), 1)
+ self.assertEqual(combined.count("第二段"), 1)
+ self.assertNotIn("<|", combined)
+
+
+# ===========================================================================
+# AgentStreamParser / parse_full_response (protocol-level helpers)
+# ===========================================================================
+
+
+class TestAgentStreamParser(unittest.TestCase):
+ """Per-request stream parsing into protocol-ready deltas."""
+
+ def test_tool_call_stream_into_openai_deltas(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser(
+ tool_call_parser="llama31", tools=_make_weather_dict_tools()
+ )
+ text = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+
+ emitted = []
+ for ch in text:
+ delta = parser.process_delta(ch)
+ if delta.tool_calls:
+ emitted.extend(delta.tool_calls)
+ end = parser.flush()
+ emitted.extend(end.tool_calls)
+
+ names = [tc["function"]["name"] for tc in emitted if tc["function"]["name"]]
+ args = "".join(tc["function"]["arguments"] for tc in emitted)
+ self.assertEqual(names, ["get_weather"])
+ self.assertEqual(json.loads(args), {"city": "Beijing"})
+ self.assertTrue(parser.has_tool_calls)
+ # All deltas must be OpenAI-protocol shaped.
+ for tc in emitted:
+ self.assertEqual(tc["type"], "function")
+ self.assertIn("index", tc)
+ self.assertIn("id", tc)
+
+ def test_reasoning_and_content_split(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser(reasoning_parser="deepseek-r1")
+ text = "hiddenanswer"
+ reasoning_parts, content_parts = [], []
+ for ch in text:
+ delta = parser.process_delta(ch)
+ if delta.reasoning_content:
+ reasoning_parts.append(delta.reasoning_content)
+ if delta.content:
+ content_parts.append(delta.content)
+ self.assertEqual("".join(reasoning_parts), "hidden")
+ self.assertEqual("".join(content_parts), "answer")
+ self.assertFalse(parser.has_tool_calls)
+
+ def test_no_parsers_is_passthrough(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser()
+ delta = parser.process_delta("hello")
+ self.assertEqual(delta.content, "hello")
+ self.assertEqual(delta.reasoning_content, "")
+ self.assertEqual(delta.tool_calls, [])
+ self.assertEqual(parser.flush().content, "")
+
+ def test_instances_do_not_share_state(self):
+ from infinilm.agents import AgentStreamParser
+
+ tools = _make_weather_dict_tools()
+ parser_a = AgentStreamParser(tool_call_parser="llama31", tools=tools)
+ parser_b = AgentStreamParser(tool_call_parser="llama31", tools=tools)
+ parser_a.process_delta('<|python_tag|>{"name":"get_weather", ')
+ delta_b = parser_b.process_delta("plain text")
+ self.assertEqual(delta_b.content, "plain text")
+ self.assertEqual(delta_b.tool_calls, [])
+
+
+class TestParseFullResponse(unittest.TestCase):
+ """One-shot parsing for non-streaming responses."""
+
+ def test_tool_call_response(self):
+ from infinilm.agents import parse_full_response
+
+ text = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+ reasoning, content, tool_calls = parse_full_response(
+ text,
+ tool_call_parser="llama31",
+ tools=_make_weather_dict_tools(),
+ )
+ self.assertIsNone(reasoning)
+ self.assertEqual(content, "")
+ self.assertEqual(len(tool_calls), 1)
+ self.assertEqual(tool_calls[0]["id"], "call_0")
+ self.assertEqual(tool_calls[0]["function"]["name"], "get_weather")
+ self.assertEqual(
+ json.loads(tool_calls[0]["function"]["arguments"]),
+ {"city": "Beijing"},
+ )
+
+ def test_reasoning_response_glm4_metadata_format(self):
+ from infinilm.agents import parse_full_response
+
+ text = 'get_weather\n{"city": "北京"}'
+ reasoning, content, tool_calls = parse_full_response(
+ text,
+ tool_call_parser="glm4-9b-0414",
+ tools=_make_weather_dict_tools(),
+ )
+ self.assertIsNone(reasoning)
+ self.assertEqual(content, "")
+ self.assertEqual(tool_calls[0]["function"]["name"], "get_weather")
+ self.assertEqual(
+ json.loads(tool_calls[0]["function"]["arguments"]),
+ {"city": "北京"},
+ )
+
+ def test_plain_text_response(self):
+ from infinilm.agents import parse_full_response
+
+ reasoning, content, tool_calls = parse_full_response(
+ "just an answer",
+ tool_call_parser="llama31",
+ tools=_make_weather_dict_tools(),
+ )
+ self.assertIsNone(reasoning)
+ self.assertEqual(content, "just an answer")
+ self.assertEqual(tool_calls, [])
+
+
+# ===========================================================================
+# Qwen3XmlDetector (Qwen3 xml tool-call format)
+# ===========================================================================
+
+
+def _qwen3_call(name: str, args: dict) -> str:
+ """Build a Qwen3-format tool_call block."""
+ open_tag = "<" + "tool_call" + ">"
+ close_tag = ""
+ return f'{open_tag}\n{{"name": "{name}", "arguments": {json.dumps(args, ensure_ascii=False)}}}\n{close_tag}'
+
+
+class TestQwen3XmlParser(unittest.TestCase):
+ def test_registration(self):
+ for alias in ("qwen3", "qwen3-30b-a3b"):
+ parser = FunctionCallParser(tool_call_parser=alias)
+ self.assertIsNotNone(parser.detector)
+
+ def test_non_stream_single_call(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ text = "I will check the weather.\n" + _qwen3_call(
+ "get_weather", {"city": "北京"}
+ )
+ normal, calls = parser.parse_non_stream(text)
+ self.assertIn("I will check the weather.", normal)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ self.assertEqual(json.loads(calls[0].parameters), {"city": "北京"})
+
+ def test_non_stream_two_calls(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ text = (
+ _qwen3_call("get_weather", {"city": "北京"})
+ + "\n"
+ + _qwen3_call("get_time", {})
+ )
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(normal, "")
+ self.assertEqual([c.name for c in calls], ["get_weather", "get_time"])
+ self.assertEqual(json.loads(calls[0].parameters), {"city": "北京"})
+ self.assertEqual(json.loads(calls[1].parameters), {})
+
+ def test_non_stream_unknown_tool_dropped(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ text = _qwen3_call("no_such_tool", {"x": 1})
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(calls, [])
+
+ def test_stream_char_by_char(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ text = "Let me see.\n" + _qwen3_call("get_weather", {"city": "Beijing"})
+ names, arg_parts, normals = [], [], []
+ for ch in text:
+ res = parser.parse_streaming_increment("", ch)
+ for c in res.calls:
+ if c.name:
+ names.append(c.name)
+ if c.parameters:
+ arg_parts.append(c.parameters)
+ if res.normal_text:
+ normals.append(res.normal_text)
+ normal_end, end_calls = parser.parse_stream_end()
+ for c in end_calls:
+ if c.name:
+ names.append(c.name)
+ if c.parameters:
+ arg_parts.append(c.parameters)
+
+ self.assertEqual(names, ["get_weather"])
+ self.assertEqual(json.loads("".join(arg_parts)), {"city": "Beijing"})
+ self.assertIn("Let me see.", "".join(normals) + normal_end)
+
+ def test_stream_two_consecutive_calls(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ text = _qwen3_call("get_weather", {"city": "北京"}) + _qwen3_call(
+ "get_time", {}
+ )
+ names, arg_parts = [], []
+ for ch in text:
+ res = parser.parse_streaming_increment("", ch)
+ for c in res.calls:
+ if c.name:
+ names.append(c.name)
+ if c.parameters:
+ arg_parts.append(c.parameters)
+ normal_end, end_calls = parser.parse_stream_end()
+ for c in end_calls:
+ if c.name:
+ names.append(c.name)
+ if c.parameters:
+ arg_parts.append(c.parameters)
+
+ self.assertEqual(names, ["get_weather", "get_time"])
+ # Arguments arrive per call; concatenate-and-split by object boundaries.
+ combined = "".join(arg_parts)
+ self.assertIn("北京", combined)
+
+ def test_stream_plain_text_not_held_forever(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ text = "Just a plain answer without any tool call."
+ parts = []
+ for ch in text:
+ res = parser.parse_streaming_increment("", ch)
+ if res.normal_text:
+ parts.append(res.normal_text)
+ normal_end, end_calls = parser.parse_stream_end()
+ self.assertEqual(end_calls, [])
+ # Only a trailing partial-token fragment may be held back.
+ emitted = "".join(parts) + normal_end
+ self.assertTrue(text.startswith(emitted) or emitted.startswith(text[:-9]))
+
+ def test_truncated_call_released_at_finish(self):
+ parser = FunctionCallParser(
+ tool_call_parser="qwen3", tools=_make_weather_dict_tools()
+ )
+ block = _qwen3_call("get_weather", {"city": "北京"})
+ truncated = block[: len(block) // 2] # cut inside the block
+ for ch in truncated:
+ parser.parse_streaming_increment("", ch)
+ normal_end, end_calls = parser.parse_stream_end()
+ # Nothing parsed as a call; the truncated block must not vanish.
+ self.assertEqual(end_calls, [])
+ self.assertTrue(normal_end)
+
+
+# ===========================================================================
+# adapt_messages (GLM-4 metadata tool-history convention)
+# ===========================================================================
+
+
+class TestAdaptMessages(unittest.TestCase):
+ """Adaptation of OpenAI tool history for GLM-4 metadata models."""
+
+ def test_no_parser_passes_messages_through(self):
+ from infinilm.agents.message_adapter import adapt_messages
+
+ messages = [
+ {"role": "user", "content": [{"type": "text", "text": "hi"}]},
+ {"role": "tool", "tool_call_id": "c1", "content": "result"},
+ ]
+ self.assertEqual(adapt_messages(messages, None), messages)
+
+ def test_other_parsers_keep_openai_roles(self):
+ from infinilm.agents.message_adapter import adapt_messages
+
+ messages = [
+ {"role": "tool", "tool_call_id": "c1", "content": "result"},
+ ]
+ self.assertEqual(adapt_messages(messages, "llama31"), messages)
+
+ def test_multimodal_message_kept_untouched(self):
+ from infinilm.agents.message_adapter import adapt_messages
+
+ message = {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "describe"},
+ {"type": "image_url", "image_url": {"url": "http://x/cat.jpg"}},
+ ],
+ }
+ self.assertEqual(adapt_messages([message], "glm4-9b-0414"), [message])
+
+ def test_tool_messages_become_observation(self):
+ from infinilm.agents.message_adapter import adapt_messages
+
+ messages = [
+ {"role": "tool", "tool_call_id": "c1", "content": '{"aqi": 42}'},
+ ]
+ self.assertEqual(
+ adapt_messages(messages, "glm4-9b-0414"),
+ [
+ {
+ "role": "observation",
+ "tool_call_id": "c1",
+ "content": '{"aqi": 42}',
+ }
+ ],
+ )
+
+ def test_assistant_tool_calls_become_metadata_messages(self):
+ from infinilm.agents.message_adapter import adapt_messages
+
+ messages = [
+ {
+ "role": "assistant",
+ "content": "Let me check.",
+ "tool_calls": [
+ {
+ "id": "c1",
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "arguments": '{"city": "Beijing"}',
+ },
+ }
+ ],
+ },
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "c2",
+ "type": "function",
+ "function": {"name": "get_time", "arguments": "{}"},
+ }
+ ],
+ },
+ ]
+ self.assertEqual(
+ adapt_messages(messages, "glm4-9b-0414"),
+ [
+ {"role": "assistant", "content": "Let me check."},
+ {
+ "role": "assistant",
+ "metadata": "get_weather",
+ "content": '{"city": "Beijing"}',
+ },
+ {
+ "role": "assistant",
+ "metadata": "get_time",
+ "content": "{}",
+ },
+ ],
+ )
+
+
+# ===========================================================================
+# AnthropicStreamConverter (OpenAI stream -> Anthropic SSE)
+# ===========================================================================
+
+
+class TestAnthropicStreamConverter(unittest.TestCase):
+ """Pure converter tests (no server needed)."""
+
+ @staticmethod
+ def _chunk(delta=None, finish_reason=None):
+ return {
+ "choices": [
+ {"index": 0, "delta": delta or {}, "finish_reason": finish_reason}
+ ]
+ }
+
+ @staticmethod
+ def _parse(events):
+ parsed = []
+ for raw in events:
+ lines = raw.splitlines()
+ event_type = lines[0].split(": ", 1)[1]
+ data = json.loads(lines[1].split(": ", 1)[1]) if len(lines) > 1 else {}
+ parsed.append((event_type, data))
+ return parsed
+
+ def test_thinking_text_tool_sequence(self):
+ from infinilm.agents.anthropic import AnthropicStreamConverter
+
+ converter = AnthropicStreamConverter(message_id="msg_x", model="m")
+ events = list(converter.begin())
+ events += converter.feed(
+ self._chunk(delta={"reasoning_content": "Let me think"})
+ )
+ events += converter.feed(self._chunk(delta={"content": "The answer"}))
+ events += converter.feed(
+ self._chunk(
+ delta={
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "function": {
+ "name": "get_weather",
+ "arguments": '{"city": "Beijing"}',
+ },
+ }
+ ]
+ }
+ )
+ )
+ events += converter.feed(self._chunk(finish_reason="tool_calls"))
+ events += list(converter.end())
+
+ parsed = self._parse(events)
+ starts = [
+ (d["index"], d["content_block"]["type"])
+ for t, d in parsed
+ if t == "content_block_start"
+ ]
+ deltas = [
+ (d["index"], d["delta"]["type"])
+ for t, d in parsed
+ if t == "content_block_delta"
+ ]
+ self.assertEqual(starts, [(0, "thinking"), (1, "text"), (2, "tool_use")])
+ self.assertEqual(
+ deltas,
+ [(0, "thinking_delta"), (1, "text_delta"), (2, "input_json_delta")],
+ )
+ message_delta = next(d for t, d in parsed if t == "message_delta")
+ self.assertEqual(message_delta["delta"]["stop_reason"], "tool_use")
+
+ def test_usage_propagated_to_message_delta(self):
+ from infinilm.agents.anthropic import AnthropicStreamConverter
+
+ converter = AnthropicStreamConverter(message_id="msg_x", model="m")
+ events = list(converter.begin())
+ events += converter.feed(self._chunk(delta={"content": "hi"}))
+ events += converter.feed(
+ self._chunk(
+ finish_reason="stop",
+ delta={},
+ )
+ )
+ # The finish chunk of the OpenAI stream carries usage.
+ events += converter.feed(
+ {
+ "choices": [{"delta": {}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 100, "completion_tokens": 7},
+ }
+ )
+ events += list(converter.end())
+
+ parsed = self._parse(events)
+ message_delta = next(d for t, d in parsed if t == "message_delta")
+ self.assertEqual(
+ message_delta["usage"], {"input_tokens": 100, "output_tokens": 7}
+ )
+
+
+# ===========================================================================
+# Request preparation and SSE rendering helpers
+# ===========================================================================
+
+
+class TestPrepareChatTemplateKwargs(unittest.TestCase):
+ def test_tools_packed_into_chat_template_kwargs(self):
+ from infinilm.agents.message_adapter import prepare_chat_template_kwargs
+
+ data = {"tools": _make_weather_dict_tools(), "tool_choice": "auto"}
+ prepare_chat_template_kwargs(data)
+ self.assertEqual(data["chat_template_kwargs"]["tools"], data["tools"])
+ self.assertEqual(data["chat_template_kwargs"]["tool_choice"], "auto")
+
+ def test_existing_kwargs_kept(self):
+ from infinilm.agents.message_adapter import prepare_chat_template_kwargs
+
+ data = {
+ "tools": _make_weather_dict_tools(),
+ "chat_template_kwargs": {"enable_thinking": False},
+ }
+ prepare_chat_template_kwargs(data)
+ self.assertIs(data["chat_template_kwargs"]["enable_thinking"], False)
+ self.assertIn("tools", data["chat_template_kwargs"])
+
+ def test_no_tools_no_tools_key(self):
+ from infinilm.agents.message_adapter import prepare_chat_template_kwargs
+
+ data = {}
+ prepare_chat_template_kwargs(data)
+ self.assertNotIn("tools", data["chat_template_kwargs"])
+
+
+class TestDeltaSseRendering(unittest.TestCase):
+ def test_delta_events_render_openai_sse_lines(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser(reasoning_parser="deepseek-r1")
+ # Long enough that the detector can rule out a pending start tag.
+ events = parser.delta_events("hello world, no tags here", "cmpl-1", "m")
+ self.assertEqual(len(events), 1)
+ self.assertTrue(events[0].startswith("data: "))
+ chunk = json.loads(events[0][6:].strip())
+ self.assertEqual(
+ chunk["choices"][0]["delta"]["content"], "hello world, no tags here"
+ )
+
+ def test_delta_events_empty_when_nothing_to_emit(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser()
+ self.assertEqual(parser.delta_events("", "cmpl-1", "m"), [])
+
+ def test_tool_call_flush_events(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser(
+ tool_call_parser="llama31", tools=_make_weather_dict_tools()
+ )
+ text = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"北京"}}'
+ events = []
+ for ch in text:
+ events.extend(parser.delta_events(ch, "cmpl-1", "m"))
+ events.extend(parser.flush_events("cmpl-1", "m"))
+
+ chunks = [json.loads(e[6:].strip()) for e in events]
+ all_calls = [
+ tc
+ for c in chunks
+ for tc in (c["choices"][0]["delta"].get("tool_calls") or [])
+ ]
+ names = [tc["function"]["name"] for tc in all_calls if tc["function"]["name"]]
+ args = "".join(tc["function"]["arguments"] for tc in all_calls)
+ self.assertEqual(names, ["get_weather"])
+ self.assertEqual(json.loads(args), {"city": "北京"})
+ self.assertTrue(parser.has_tool_calls)
+
+
+class TestOpenaiSseLineParsing(unittest.TestCase):
+ def test_parse_data_line(self):
+ from infinilm.agents.anthropic import parse_openai_sse_line
+
+ chunk = parse_openai_sse_line('data: {"choices": []}\n\n')
+ self.assertEqual(chunk, {"choices": []})
+
+ def test_non_data_lines_return_none(self):
+ from infinilm.agents.anthropic import parse_openai_sse_line
+
+ self.assertIsNone(parse_openai_sse_line("data: [DONE]\n\n"))
+ self.assertIsNone(parse_openai_sse_line(": keepalive\n\n"))
+ self.assertIsNone(parse_openai_sse_line("data: {invalid json}\n\n"))
+
+ def test_anthropic_error_body(self):
+ from infinilm.agents.anthropic import anthropic_error_body
+
+ body = anthropic_error_body("boom")
+ self.assertEqual(body["type"], "error")
+ self.assertEqual(body["error"]["type"], "invalid_request_error")
+ self.assertEqual(body["error"]["message"], "boom")
+
+
+class TestConvertOpenaiSseStream(unittest.TestCase):
+ def test_full_stream_conversion(self):
+ import asyncio
+
+ from infinilm.agents.anthropic import convert_openai_sse_stream
+
+ def sse(delta=None, finish_reason=None):
+ chunk = {
+ "choices": [
+ {"index": 0, "delta": delta or {}, "finish_reason": finish_reason}
+ ]
+ }
+ return f"data: {json.dumps(chunk)}\n\n"
+
+ async def fake_stream():
+ yield sse(delta={"reasoning_content": "think"})
+ yield sse(delta={"content": "answer"})
+ yield "data: [DONE]\n\n"
+
+ async def run():
+ events = []
+ async for event in convert_openai_sse_stream(
+ fake_stream(), message_id="msg_x", model="m"
+ ):
+ events.append(event)
+ return events
+
+ events = asyncio.run(run())
+ joined = "".join(events)
+ for fragment in (
+ "message_start",
+ '"type": "thinking"',
+ "thinking_delta",
+ '"type": "text"',
+ "text_delta",
+ "content_block_stop",
+ '"stop_reason": "end_turn"',
+ "message_stop",
+ ):
+ self.assertIn(fragment, joined)
+
+
+# ===========================================================================
+# parse_arguments utility
+# ===========================================================================
+
+
+class TestParseArguments(unittest.TestCase):
+ """Tests for the parse_arguments utility function."""
+
+ def test_string_json(self):
+ args, ok = parse_arguments('{"a": 1}')
+ self.assertTrue(ok)
+ self.assertEqual(args, {"a": 1})
+
+ def test_dict_passthrough(self):
+ args, ok = parse_arguments({"b": 2})
+ self.assertTrue(ok)
+ self.assertEqual(args, {"b": 2})
+
+ def test_literal_eval(self):
+ args, ok = parse_arguments("{'c': 3}")
+ self.assertTrue(ok)
+ self.assertEqual(args, {"c": 3})
+
+ def test_list_input(self):
+ args, ok = parse_arguments("[1, 2, 3]")
+ self.assertTrue(ok)
+ self.assertEqual(args, [1, 2, 3])
+
+ def test_number_string(self):
+ args, ok = parse_arguments("42")
+ self.assertTrue(ok)
+ self.assertEqual(args, 42)
+
+ def test_boolean_string(self):
+ args, ok = parse_arguments("true")
+ self.assertTrue(ok)
+ self.assertEqual(args, True)
+
+ def test_escaped_string_value(self):
+ args, ok = parse_arguments('{"key": "value with \\"quotes\\""}')
+ self.assertTrue(ok)
+ parsed = json.loads('{"key": "value with \\"quotes\\""}')
+ self.assertEqual(args, parsed)
+
+
+# ===========================================================================
+# StreamingParseResult
+# ===========================================================================
+
+
+class TestStreamingParseResult(unittest.TestCase):
+ """Tests for StreamingParseResult dataclass."""
+
+ def test_defaults(self):
+ r = StreamingParseResult()
+ self.assertEqual(r.normal_text, "")
+ self.assertEqual(r.calls, [])
+
+ def test_with_calls(self):
+ r = StreamingParseResult(
+ normal_text="hi",
+ calls=[ToolCallItem(tool_index=0, name="test", parameters="{}")],
+ )
+ self.assertEqual(len(r.calls), 1)
+ self.assertEqual(r.calls[0].name, "test")
+
+ def test_multiple_calls(self):
+ r = StreamingParseResult(
+ normal_text="",
+ calls=[
+ ToolCallItem(tool_index=0, name="f1", parameters='{"a":1}'),
+ ToolCallItem(tool_index=1, name="f2", parameters='{"b":2}'),
+ ],
+ )
+ self.assertEqual(len(r.calls), 2)
+ self.assertEqual(r.calls[1].name, "f2")
+
+ def test_no_name_call(self):
+ """ToolCallItem can have name=None (parameter-only streaming events)."""
+ r = StreamingParseResult(
+ normal_text="",
+ calls=[
+ ToolCallItem(tool_index=0, name=None, parameters='{"city":"NYC"}'),
+ ],
+ )
+ self.assertIsNone(r.calls[0].name)
+ self.assertEqual(r.calls[0].parameters, '{"city":"NYC"}')
+
+
+# ===========================================================================
+# Integration Tests
+# ===========================================================================
+
+
+class TestIntegration(unittest.TestCase):
+ """Tests that simulate the inference_server pipeline."""
+
+ def test_reasoning_then_tool_call_non_stream_glm45(self):
+ """Extract reasoning via glm45, then parse tool calls from remaining text."""
+ # Step 1: Extract reasoning
+ reasoning_parser = ReasoningParser(reasoning_parser_name="glm45")
+ full_text = (
+ "thinkI should call get_weather for Beijing."
+ "/thinkChecking weather...\n"
+ "get_weather\n"
+ "city\nBeijing\n"
+ ""
+ )
+ reasoning_results = reasoning_parser.extract_reasoning_content(full_text)
+
+ self.assertEqual(len(reasoning_results), 2)
+ rc = reasoning_results[0]
+ nt = reasoning_results[1]
+ self.assertEqual(rc.reasoning_content, "I should call get_weather for Beijing.")
+ self.assertTrue(rc.complete)
+ self.assertIn("Checking weather...", nt.normal_text)
+
+ # Step 2: Parse tool calls from the normal_text part
+ tools = _make_weather_tools()
+ tool_parser = FunctionCallParser(tool_call_parser="glm", tools=tools)
+ normal, calls = tool_parser.parse_non_stream(nt.normal_text)
+ self.assertIn("Checking weather...", normal)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "Beijing"})
+
+ def test_reasoning_then_tool_call_non_stream_think(self):
+ """Extract reasoning via think tags, then parse tool calls via Llama."""
+ reasoning_parser = ReasoningParser(reasoning_parser_name="think")
+ full_text = (
+ "I should call get_time.The time is now.\n"
+ '<|python_tag|>{"name":"get_time", "arguments":{}}'
+ )
+ results = reasoning_parser.extract_reasoning_content(full_text)
+ self.assertEqual(len(results), 2)
+ self.assertEqual(results[0].reasoning_content, "I should call get_time.")
+ self.assertTrue(results[0].complete)
+
+ remaining = results[1].normal_text
+ tools = _make_weather_tools()
+ tool_parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ normal, calls = tool_parser.parse_non_stream(remaining)
+ self.assertIn("The time is now.", normal)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_time")
+
+ def test_llama31_with_raw_dict_tools(self):
+ """Full pipeline: raw dict tools from HTTP JSON -> parse tool calls."""
+ # Tools from HTTP request body (plain dicts, not Tool objects)
+ raw_tools = _make_weather_dict_tools()
+
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=raw_tools)
+ text = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"London"}}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "London"})
+
+ def test_llama31_streaming_with_raw_dict_tools(self):
+ """Streaming with raw dict tools."""
+ raw_tools = _make_weather_dict_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=raw_tools)
+ call_text = '<|python_tag|>{"name":"get_time", "arguments":{}}'
+ result = parser.parse_streaming_increment("", call_text, raw_tools)
+ names = [c.name for c in result.calls if c.name]
+ self.assertIn("get_time", names)
+
+
+class TestIntegrationEdgeCases(unittest.TestCase):
+ """Edge-case integration tests."""
+
+ def test_empty_reasoning_then_tool_call(self):
+ """Empty reasoning (no output) followed by tool call still parsed."""
+ parser = FunctionCallParser(
+ tool_call_parser="llama31", tools=_make_weather_tools()
+ )
+ text = 'Hello<|python_tag|>{"name":"get_time", "arguments":{}}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertIn("Hello", normal)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_time")
+
+ def test_only_tool_call_no_text(self):
+ """Pure tool call with no surrounding text."""
+ tools = _make_weather_tools()
+ parser = FunctionCallParser(tool_call_parser="llama31", tools=tools)
+ text = '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Tokyo"}}'
+ normal, calls = parser.parse_non_stream(text)
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0].name, "get_weather")
+ params = json.loads(calls[0].parameters)
+ self.assertEqual(params, {"city": "Tokyo"})
+
+
+# ===========================================================================
+# Main
+# ===========================================================================
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/agents/test_llm_agents.py b/test/agents/test_llm_agents.py
new file mode 100644
index 000000000..4d8929876
--- /dev/null
+++ b/test/agents/test_llm_agents.py
@@ -0,0 +1,248 @@
+"""
+Regression test: ``AsyncLLMEngine.add_chat_request()`` must forward
+``chat_template_kwargs`` (in particular tool definitions) through to
+``apply_chat_template()`` so the tools actually enter the prompt.
+
+The test runs against the real ``infinilm/llm/llm.py`` code; engine internals
+that require the native stack are stubbed only when the real modules cannot
+be imported (CPU-only environments).
+"""
+
+import enum
+import sys
+import types
+import unittest
+from pathlib import Path
+
+# Ensure infinilm is importable when running tests directly. The Python
+# sources live under /python.
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+PYTHON_ROOT = PROJECT_ROOT / "python"
+for _path in (str(PYTHON_ROOT), str(PROJECT_ROOT)):
+ if _path not in sys.path:
+ sys.path.insert(0, _path)
+
+
+def _namespace_stub(name: str, path: str):
+ mod = sys.modules.get(name)
+ if mod is None or not hasattr(mod, "__path__"):
+ mod = types.ModuleType(name)
+ sys.modules[name] = mod
+ mod.__path__ = [path]
+ return mod
+
+
+def _stub(name: str, **attrs):
+ mod = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(mod, key, value)
+ sys.modules[name] = mod
+ return mod
+
+
+try:
+ from infinilm.llm.llm import AsyncLLMEngine, LLMEngine
+ from infinilm.llm.sampling_params import SamplingParams
+
+ _STUBBED = False
+except Exception:
+ # CPU-only environment without the compiled engine: stand in for the
+ # native-dependent modules so the real llm.py logic can still run.
+
+ class FinishReason(enum.Enum):
+ EOS_TOKEN = "eos_token"
+ STOP_STRING = "stop_string"
+ STOP = "stop"
+ LENGTH = "length"
+ CANCELED = "canceled"
+ TIMEOUT = "timeout"
+ ERROR = "error"
+
+ class SamplingParams: # minimal mirror of the real dataclass
+ def __init__(self, **kwargs):
+ self.kwargs = dict(kwargs)
+ for key, value in kwargs.items():
+ setattr(self, key, value)
+
+ def clone(self):
+ return SamplingParams(**self.kwargs)
+
+ class InferenceRequest:
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+
+ @property
+ def output_queue(self):
+ return None
+
+ def _empty_mm_inputs(messages):
+ return {
+ "images": [],
+ "image_urls": [],
+ "videos": [],
+ "video_urls": [],
+ "audios": [],
+ "audio_urls": [],
+ }
+
+ if "janus" not in sys.modules:
+ try:
+ import janus # noqa: F401
+ except Exception:
+ _stub("janus", Queue=object)
+
+ _namespace_stub("infinilm", str(PYTHON_ROOT / "infinilm"))
+ _namespace_stub("infinilm.llm", str(PYTHON_ROOT / "infinilm" / "llm"))
+ _namespace_stub(
+ "infinilm.llm.model_runner",
+ str(PYTHON_ROOT / "infinilm" / "llm" / "model_runner"),
+ )
+ _namespace_stub("infinilm.multimodal", str(PYTHON_ROOT / "infinilm" / "multimodal"))
+ _namespace_stub("infinilm.config", str(PYTHON_ROOT / "infinilm" / "config"))
+
+ _stub("infinilm.config.engine_config", EngineConfig=object)
+ _stub("infinilm.config.kv_transfer", KVTransferConfig=object)
+ _stub(
+ "infinilm.infer_engine",
+ model_uses_mamba_cache=lambda config: False,
+ read_hf_config=lambda path: {},
+ )
+ _stub(
+ "infinilm.kv_connector",
+ KVConnectorFactory=object,
+ KVConnectorRole=object,
+ )
+ _stub("infinilm.llm.model_runner.model_runner", ModelRunner=object)
+ _stub(
+ "infinilm.llm.request",
+ FinishReason=FinishReason,
+ InferenceRequest=InferenceRequest,
+ RequestOutput=object,
+ TokenOutput=object,
+ )
+ _stub("infinilm.llm.sampling_params", SamplingParams=SamplingParams)
+ _stub("infinilm.llm.scheduler", Scheduler=object, SchedulerOutput=object)
+ _stub(
+ "infinilm.llm.static_scheduler",
+ StaticScheduler=object,
+ StaticSchedulerOutput=object,
+ )
+ _stub(
+ "infinilm.multimodal.multimodal",
+ resolve_multimodal_inputs=_empty_mm_inputs,
+ )
+
+ from infinilm.llm.llm import AsyncLLMEngine, LLMEngine
+
+ _STUBBED = True
+
+
+def _make_weather_tools():
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "description": "Get weather for a city",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ "required": ["city"],
+ },
+ },
+ }
+ ]
+
+
+class _FakeInputIds:
+ """Minimal stand-in for a token-id tensor."""
+
+ def flatten(self):
+ return self
+
+ def tolist(self):
+ return [1, 2, 3]
+
+
+class TestChatTemplateKwargsForwarding(unittest.TestCase):
+ """P1-1: tools sent via chat_template_kwargs must reach the template."""
+
+ def _make_engine(self, captured: dict):
+ class FakeProcessor:
+ def apply_chat_template(
+ self, conversation, add_generation_prompt, tokenize, **kwargs
+ ):
+ captured.update(kwargs)
+ return "PROMPT_WITH_TOOLS" if "tools" in kwargs else "PROMPT"
+
+ def __call__(self, prompt, images=None, videos=None, audios=None, **kw):
+ return {"input_ids": _FakeInputIds()}
+
+ def get_mm_token_index_list(self, *args, **kwargs):
+ return {}
+
+ class FakeScheduler:
+ def __init__(self):
+ self.added = []
+
+ def add_request(self, req):
+ self.added.append(req)
+
+ llm_engine = object.__new__(LLMEngine)
+ llm_engine.processor = FakeProcessor()
+ llm_engine.scheduler = FakeScheduler()
+ llm_engine.eos_token_ids = [2]
+
+ engine = object.__new__(AsyncLLMEngine)
+ engine.engine = llm_engine
+ engine.config = types.SimpleNamespace(max_tokens=64)
+ return engine
+
+ def test_tools_reach_chat_template(self):
+ captured = {}
+ engine = self._make_engine(captured)
+ tools = _make_weather_tools()
+
+ request = engine.add_chat_request(
+ messages=[{"role": "user", "content": "weather in Beijing?"}],
+ sampling_params=SamplingParams(max_tokens=32),
+ request_id="cmpl-tools",
+ chat_template_kwargs={"tools": tools, "tool_choice": "auto"},
+ )
+
+ self.assertEqual(request.prompt, "PROMPT_WITH_TOOLS")
+ self.assertIn("tools", captured)
+ self.assertEqual(captured["tools"], tools)
+ self.assertEqual(captured["tool_choice"], "auto")
+
+ def test_no_tools_no_template_kwargs(self):
+ captured = {}
+ engine = self._make_engine(captured)
+
+ request = engine.add_chat_request(
+ messages=[{"role": "user", "content": "hello"}],
+ sampling_params=SamplingParams(max_tokens=32),
+ request_id="cmpl-plain",
+ )
+
+ self.assertEqual(request.prompt, "PROMPT")
+ self.assertNotIn("tools", captured)
+
+ def test_add_request_accepts_chat_template_kwargs(self):
+ """The lower-level add_request() also forwards chat_template_kwargs."""
+ captured = {}
+ engine = self._make_engine(captured)
+ tools = _make_weather_tools()
+
+ engine.add_request(
+ messages=[{"role": "user", "content": "hi"}],
+ sampling_params=SamplingParams(max_tokens=32),
+ request_id="cmpl-lowlevel",
+ chat_template_kwargs={"tools": tools},
+ )
+
+ self.assertEqual(captured.get("tools"), tools)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/agents/test_server_agents.py b/test/agents/test_server_agents.py
new file mode 100644
index 000000000..8b6aff207
--- /dev/null
+++ b/test/agents/test_server_agents.py
@@ -0,0 +1,585 @@
+"""
+Regression tests for the agent-related server logic in inference_server.py.
+
+The server module normally pulls in the native engine stack; in CPU-only
+environments lightweight stand-ins are registered for those engine modules so
+the pure-Python request/response shaping code can still be exercised.
+"""
+
+import asyncio
+import enum
+import json
+import sys
+import types
+import unittest
+from pathlib import Path
+
+# Ensure infinilm is importable when running tests directly. The Python
+# sources live under /python.
+PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+PYTHON_ROOT = PROJECT_ROOT / "python"
+for _path in (str(PYTHON_ROOT), str(PROJECT_ROOT)):
+ if _path not in sys.path:
+ sys.path.insert(0, _path)
+
+
+def _ensure_infinilm_stub():
+ """Make ``infinilm`` importable without the native extension."""
+ try:
+ import infinilm # noqa: F401
+ except Exception:
+ sys.modules.pop("infinilm", None)
+ stub = types.ModuleType("infinilm")
+ stub.__path__ = [str(PYTHON_ROOT / "infinilm")]
+ sys.modules["infinilm"] = stub
+
+
+def _install_engine_stubs():
+ """Register stand-ins for engine modules that require the native stack."""
+
+ class FinishReason(enum.Enum):
+ EOS_TOKEN = "eos_token"
+ STOP_STRING = "stop_string"
+ STOP = "stop"
+ LENGTH = "length"
+ CANCELED = "canceled"
+ TIMEOUT = "timeout"
+ ERROR = "error"
+
+ class SamplingParams:
+ def __init__(self, **kwargs):
+ self.kwargs = dict(kwargs)
+ for key, value in kwargs.items():
+ setattr(self, key, value)
+
+ def clone(self):
+ return SamplingParams(**self.kwargs)
+
+ class AsyncLLMEngine: # never instantiated in these tests
+ def __init__(self, *args, **kwargs):
+ raise RuntimeError("engine stub must not be instantiated")
+
+ class KVTransferConfig:
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+
+ class BaseConfig: # only referenced by main()
+ pass
+
+ def _module(name, **attrs):
+ mod = types.ModuleType(name)
+ for key, value in attrs.items():
+ setattr(mod, key, value)
+ sys.modules[name] = mod
+
+ class SchedulerOutput: # only used in isinstance checks
+ pass
+
+ class StaticSchedulerOutput: # only used in isinstance checks
+ pass
+
+ _module("infinilm.base_config", BaseConfig=BaseConfig)
+ _module("infinilm.config", KVTransferConfig=KVTransferConfig)
+ _module(
+ "infinilm.llm",
+ AsyncLLMEngine=AsyncLLMEngine,
+ FinishReason=FinishReason,
+ SamplingParams=SamplingParams,
+ )
+ _module(
+ "infinilm.llm.scheduler",
+ SchedulerOutput=SchedulerOutput,
+ )
+ _module(
+ "infinilm.llm.static_scheduler",
+ StaticSchedulerOutput=StaticSchedulerOutput,
+ )
+ _module(
+ "infinilm.moe_config",
+ configure_moe_ep_backend=lambda *args, **kwargs: ("disabled", 1),
+ )
+
+
+_ensure_infinilm_stub()
+try:
+ from infinilm.llm import FinishReason # noqa: F401
+except Exception:
+ _install_engine_stubs()
+
+try:
+ from infinilm.server.inference_server import InferenceServer
+
+ SERVER_IMPORT_ERROR = None
+except Exception as exc: # pragma: no cover - environment dependent
+ InferenceServer = None
+ SERVER_IMPORT_ERROR = exc
+
+
+def _make_server(**kwargs) -> "InferenceServer":
+ kwargs.setdefault("model_path", "dummy-model")
+ return InferenceServer(**kwargs)
+
+
+def _openai_chunk(delta=None, finish_reason=None) -> str:
+ chunk = {
+ "id": "cmpl-test",
+ "object": "chat.completion.chunk",
+ "created": 0,
+ "model": "dummy",
+ "choices": [
+ {
+ "index": 0,
+ "delta": delta or {},
+ "logprobs": None,
+ "finish_reason": finish_reason,
+ }
+ ],
+ }
+ return f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n"
+
+
+def _parse_sse_events(raw_events):
+ """Parse 'event: X\\ndata: {...}\\n\\n' strings into (type, data) pairs."""
+ parsed = []
+ for raw in raw_events:
+ lines = raw.splitlines()
+ event_type = lines[0].split(": ", 1)[1]
+ data = json.loads(lines[1].split(": ", 1)[1]) if len(lines) > 1 else {}
+ parsed.append((event_type, data))
+ return parsed
+
+
+@unittest.skipIf(
+ InferenceServer is None,
+ f"inference_server not importable in this environment: {SERVER_IMPORT_ERROR}",
+)
+@unittest.skipIf(
+ InferenceServer is None,
+ f"inference_server not importable in this environment: {SERVER_IMPORT_ERROR}",
+)
+class TestPerRequestParsers(unittest.TestCase):
+ """P1-2: parser state must not be shared between concurrent requests."""
+
+ def test_stream_parsers_are_independent(self):
+ from infinilm.agents import AgentStreamParser
+
+ tools = [
+ {
+ "type": "function",
+ "function": {"name": "get_weather", "parameters": {}},
+ }
+ ]
+ parser_a = AgentStreamParser(tool_call_parser="llama31", tools=tools)
+ parser_b = AgentStreamParser(tool_call_parser="llama31", tools=tools)
+ parser_a.process_delta('<|python_tag|>{"name":"get_weather", ')
+ # Resetting B's detector must not touch A's buffered state.
+ parser_b._tool_call_parser.detector.clear()
+ delta = parser_a.flush()
+ self.assertTrue(
+ parser_a._tool_call_parser.detector._buffer or delta.tool_calls,
+ "request A state was clobbered by B",
+ )
+
+ def test_no_shared_parser_instances_on_server(self):
+ server = _make_server(tool_call_parser="glm", reasoning_parser="think")
+ self.assertFalse(hasattr(server, "_tool_call_parser_instance"))
+ self.assertFalse(hasattr(server, "_reasoning_parser_instance"))
+
+ def test_disabled_parsers_pass_text_through(self):
+ from infinilm.agents import AgentStreamParser
+
+ parser = AgentStreamParser()
+ delta = parser.process_delta("plain text")
+ self.assertEqual(delta.content, "plain text")
+ self.assertEqual(delta.reasoning_content, "")
+ self.assertEqual(delta.tool_calls, [])
+ self.assertFalse(parser.has_tool_calls)
+
+
+try:
+ from infinilm.processors.basic_llm_processor import BasicLLMProcessor
+
+ PROCESSOR_IMPORT_ERROR = None
+except Exception as exc: # pragma: no cover - environment dependent
+ BasicLLMProcessor = None
+ PROCESSOR_IMPORT_ERROR = exc
+
+
+@unittest.skipIf(
+ BasicLLMProcessor is None,
+ f"basic_llm_processor not importable in this environment: {PROCESSOR_IMPORT_ERROR}",
+)
+class TestConversationNormalization(unittest.TestCase):
+ """Engine-level content handling (moved out of the HTTP layer)."""
+
+ def test_text_only_list_joined_into_string(self):
+ conversation = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "part1"},
+ {"type": "text", "text": "part2"},
+ ],
+ }
+ ]
+ self.assertEqual(
+ BasicLLMProcessor.normalize_conversation(conversation),
+ [{"role": "user", "content": "part1part2"}],
+ )
+
+ def test_string_content_untouched(self):
+ conversation = [{"role": "user", "content": "hello"}]
+ self.assertEqual(
+ BasicLLMProcessor.normalize_conversation(conversation), conversation
+ )
+
+ def test_metadata_and_observation_messages_untouched(self):
+ conversation = [
+ {"role": "assistant", "metadata": "get_weather", "content": "{}"},
+ {"role": "observation", "content": '{"aqi": 42}'},
+ ]
+ self.assertEqual(
+ BasicLLMProcessor.normalize_conversation(conversation), conversation
+ )
+
+ def test_non_text_parts_rejected_for_text_models(self):
+ conversation = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "describe"},
+ {"type": "image_url", "image_url": {"url": "http://x/y.jpg"}},
+ ],
+ }
+ ]
+ with self.assertRaises(ValueError):
+ BasicLLMProcessor.normalize_conversation(conversation)
+
+
+@unittest.skipIf(
+ InferenceServer is None,
+ f"inference_server not importable in this environment: {SERVER_IMPORT_ERROR}",
+)
+class TestAnthropicStream(unittest.IsolatedAsyncioTestCase):
+ """P1-5: content blocks need explicit types and monotonic indices."""
+
+ async def _run_stream(self, chunks):
+ server = _make_server()
+
+ async def fake_stream_chat(request_id, data, http_request):
+ for chunk in chunks:
+ yield chunk
+ yield "data: [DONE]\n\n"
+
+ server._stream_chat = fake_stream_chat
+ raw_events = []
+ async for event in server._anthropic_stream("msg_test", {}, None):
+ raw_events.append(event)
+ return _parse_sse_events(raw_events)
+
+ async def test_thinking_text_tool_sequence(self):
+ """Reasoning, then text, then a tool call: three blocks, 0 -> 1 -> 2."""
+ chunks = [
+ _openai_chunk(delta={"reasoning_content": "Let me think"}),
+ _openai_chunk(delta={"content": "The weather is"}),
+ _openai_chunk(
+ delta={
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "get_weather", "arguments": ""},
+ }
+ ]
+ }
+ ),
+ _openai_chunk(
+ delta={
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {"arguments": '{"city":"Beijing"}'},
+ }
+ ]
+ }
+ ),
+ _openai_chunk(finish_reason="tool_calls"),
+ ]
+ events = await self._run_stream(chunks)
+
+ starts = [
+ (data["index"], data["content_block"]["type"])
+ for event_type, data in events
+ if event_type == "content_block_start"
+ ]
+ deltas = [
+ (data["index"], data["delta"]["type"])
+ for event_type, data in events
+ if event_type == "content_block_delta"
+ ]
+ stops = [
+ data["index"]
+ for event_type, data in events
+ if event_type == "content_block_stop"
+ ]
+
+ self.assertEqual(starts, [(0, "thinking"), (1, "text"), (2, "tool_use")])
+ self.assertEqual(
+ deltas,
+ [(0, "thinking_delta"), (1, "text_delta"), (2, "input_json_delta")],
+ )
+ self.assertEqual(stops, [0, 1, 2])
+
+ # Every delta must target a block declared with the matching type.
+ declared = dict(starts)
+ delta_type_for_block = {
+ "thinking": "thinking_delta",
+ "text": "text_delta",
+ "tool_use": "input_json_delta",
+ }
+ for index, delta_type in deltas:
+ self.assertEqual(delta_type, delta_type_for_block[declared[index]])
+
+ message_delta = next(
+ data for event_type, data in events if event_type == "message_delta"
+ )
+ self.assertEqual(message_delta["delta"]["stop_reason"], "tool_use")
+ event_types = [event_type for event_type, _ in events]
+ self.assertEqual(event_types[0], "message_start")
+ self.assertEqual(event_types[-1], "message_stop")
+
+ async def test_tool_block_index_not_reused(self):
+ """Regression: tool_use used to reuse index 0 after thinking/text."""
+ chunks = [
+ _openai_chunk(delta={"reasoning_content": "hmm"}),
+ _openai_chunk(delta={"content": "answer"}),
+ _openai_chunk(
+ delta={
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "f", "arguments": "{}"},
+ }
+ ]
+ }
+ ),
+ _openai_chunk(finish_reason="tool_calls"),
+ ]
+ events = await self._run_stream(chunks)
+ start_indices = [
+ data["index"]
+ for event_type, data in events
+ if event_type == "content_block_start"
+ ]
+ # Indices must be unique and monotonically increasing.
+ self.assertEqual(start_indices, sorted(set(start_indices)))
+ self.assertEqual(len(start_indices), 3)
+
+ async def test_two_parallel_tool_calls(self):
+ """Two tool calls become two separate tool_use blocks."""
+ chunks = [
+ _openai_chunk(
+ delta={
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_0",
+ "type": "function",
+ "function": {"name": "get_weather", "arguments": "{}"},
+ }
+ ]
+ }
+ ),
+ _openai_chunk(
+ delta={
+ "tool_calls": [
+ {
+ "index": 1,
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "get_time", "arguments": "{}"},
+ }
+ ]
+ }
+ ),
+ _openai_chunk(finish_reason="tool_calls"),
+ ]
+ events = await self._run_stream(chunks)
+ starts = [
+ (
+ data["index"],
+ data["content_block"]["type"],
+ data["content_block"]["name"],
+ )
+ for event_type, data in events
+ if event_type == "content_block_start"
+ ]
+ self.assertEqual(
+ starts,
+ [(0, "tool_use", "get_weather"), (1, "tool_use", "get_time")],
+ )
+
+ async def test_text_only_stream(self):
+ chunks = [
+ _openai_chunk(delta={"content": "Hello "}),
+ _openai_chunk(delta={"content": "world"}),
+ _openai_chunk(finish_reason="stop"),
+ ]
+ events = await self._run_stream(chunks)
+ starts = [
+ (data["index"], data["content_block"]["type"])
+ for event_type, data in events
+ if event_type == "content_block_start"
+ ]
+ deltas = [
+ (data["index"], data["delta"].get("text"))
+ for event_type, data in events
+ if event_type == "content_block_delta"
+ ]
+ self.assertEqual(starts, [(0, "text")])
+ self.assertEqual(deltas, [(0, "Hello "), (0, "world")])
+ message_delta = next(
+ data for event_type, data in events if event_type == "message_delta"
+ )
+ self.assertEqual(message_delta["delta"]["stop_reason"], "end_turn")
+
+
+@unittest.skipIf(
+ InferenceServer is None,
+ f"inference_server not importable in this environment: {SERVER_IMPORT_ERROR}",
+)
+class TestConcurrentStreams(unittest.IsolatedAsyncioTestCase):
+ """P1-2 regression: two interleaved streams must not share parser state."""
+
+ async def test_two_interleaved_tool_call_streams(self):
+ from infinilm.llm import FinishReason
+
+ call_text_a = (
+ '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Beijing"}}'
+ )
+ call_text_b = (
+ '<|python_tag|>{"name":"get_weather", "arguments":{"city":"Shanghai"}}'
+ )
+
+ class FakeRequest:
+ def __init__(self, request_id, texts):
+ self.request_id = request_id
+ self._texts = texts
+ self._finished = False
+
+ def is_finished(self):
+ return self._finished
+
+ def get_prompt_length(self):
+ return 10
+
+ def get_num_generated_tokens(self):
+ return len(self._texts)
+
+ def get_total_length(self):
+ return 10 + len(self._texts)
+
+ class FakeInnerEngine:
+ eos_token_ids = []
+
+ class FakeEngine:
+ def __init__(self, streams):
+ self.engine = FakeInnerEngine()
+ self._streams = streams
+
+ def add_chat_request(self, *, request_id, **kwargs):
+ return FakeRequest(request_id, self._streams[request_id])
+
+ async def stream_request(self, req, timeout=None, request_timeout=None):
+ for i, text in enumerate(req._texts):
+ await asyncio.sleep(0) # force interleaving with other streams
+ finished = i == len(req._texts) - 1
+ yield types.SimpleNamespace(
+ token_id=0,
+ token_text=text,
+ finished=finished,
+ finish_reason=FinishReason.EOS_TOKEN if finished else None,
+ )
+ req._finished = True
+
+ def add_aborted_req(self, req, reason):
+ pass
+
+ class FakeHttpRequest:
+ async def is_disconnected(self):
+ return False
+
+ # Stream the two tool calls in small interleaved chunks.
+ def split(text, n):
+ step = -(-len(text) // n)
+ return [
+ text[i * step : (i + 1) * step] for i in range(n) if text[i * step :]
+ ]
+
+ streams = {"req-a": split(call_text_a, 4), "req-b": split(call_text_b, 4)}
+
+ server = _make_server(tool_call_parser="llama31")
+ server.engine = FakeEngine(streams)
+
+ data = {
+ "messages": [{"role": "user", "content": "weather?"}],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ },
+ },
+ }
+ ],
+ "chat_template_kwargs": {},
+ }
+
+ async def collect(request_id):
+ events = []
+ async for raw in server._stream_chat(
+ request_id, dict(data), FakeHttpRequest()
+ ):
+ events.append(raw)
+ return events
+
+ events_a, events_b = await asyncio.gather(collect("req-a"), collect("req-b"))
+
+ def tool_calls_from(events):
+ names, arg_parts, content_parts = [], [], []
+ for raw in events:
+ if not raw.startswith("data: ") or raw.startswith("data: [DONE]"):
+ continue
+ chunk = json.loads(raw[6:].strip())
+ delta = chunk["choices"][0].get("delta", {})
+ if delta.get("content"):
+ content_parts.append(delta["content"])
+ for tc in delta.get("tool_calls") or []:
+ fn = tc.get("function", {})
+ if fn.get("name"):
+ names.append(fn["name"])
+ if fn.get("arguments"):
+ arg_parts.append(fn["arguments"])
+ return names, "".join(arg_parts), "".join(content_parts)
+
+ names_a, args_a, content_a = tool_calls_from(events_a)
+ names_b, args_b, content_b = tool_calls_from(events_b)
+
+ # Each stream must see exactly its own complete tool call; nothing
+ # may leak into plain content.
+ self.assertEqual(names_a, ["get_weather"])
+ self.assertEqual(json.loads(args_a), {"city": "Beijing"})
+ self.assertEqual(content_a, "")
+ self.assertEqual(names_b, ["get_weather"])
+ self.assertEqual(json.loads(args_b), {"city": "Shanghai"})
+ self.assertEqual(content_b, "")
+
+
+if __name__ == "__main__":
+ unittest.main()