Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 39 additions & 1 deletion src/ghidra_deep_agent/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@
import sys
import tomllib
import uuid
from collections.abc import AsyncIterator, Iterator
from pathlib import Path
from typing import Any, cast

from langchain_core.embeddings import Embeddings
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage
from langchain_core.messages import AIMessage, AIMessageChunk
from langchain_core.messages.ai import UsageMetadata, subtract_usage
from langchain_core.outputs import ChatGenerationChunk
from langchain_deepseek import ChatDeepSeek
from pydantic import SecretStr

Expand Down Expand Up @@ -42,6 +45,41 @@ def _get_request_payload(

return payload

# Some backends behind the OpenCode gateway attach the *running* usage total
# to every streamed chunk instead of once at the end. LangChain sums chunk
# usage when merging, so one 34s call was reported as 53M input / 1.4M
# output tokens (~405 chunks x a 131k prompt). Rewriting each chunk's usage
# as the delta from the previous total sums correctly either way.
def _stream(self, *args: Any, **kwargs: Any) -> Iterator[ChatGenerationChunk]:
seen: UsageMetadata | None = None
for chunk in super()._stream(*args, **kwargs):
seen = _usage_to_delta(chunk, seen)
yield chunk

async def _astream(
self, *args: Any, **kwargs: Any
) -> AsyncIterator[ChatGenerationChunk]:
seen: UsageMetadata | None = None
async for chunk in super()._astream(*args, **kwargs):
seen = _usage_to_delta(chunk, seen)
yield chunk


def _usage_to_delta(
chunk: ChatGenerationChunk, seen: UsageMetadata | None
) -> UsageMetadata | None:
"""Replace ``chunk``'s cumulative usage with the delta since ``seen``.

Returns the cumulative usage so far, to pass in with the next chunk.
"""
message = chunk.message
if not isinstance(message, AIMessageChunk) or not message.usage_metadata:
return seen
usage = message.usage_metadata
if seen is not None:
message.usage_metadata = subtract_usage(usage, seen)
return usage


def build_embeddings(embed_string: str) -> Embeddings:
"""Return an embeddings instance for the given provider:model string.
Expand Down
45 changes: 44 additions & 1 deletion tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@

import pytest
from langchain_anthropic import ChatAnthropic
from langchain_core.messages import HumanMessage
from langchain_core.messages import AIMessageChunk, HumanMessage
from langchain_core.outputs import ChatGenerationChunk
from langchain_openai import ChatOpenAI

from ghidra_deep_agent.models import (
_UNPROFILED_ANTHROPIC_MAX_TOKENS,
_ChatDeepSeekFixed,
_usage_to_delta,
build_model,
)

Expand Down Expand Up @@ -250,3 +252,44 @@ def test_reasoning_effort_is_ignored_with_a_warning_elsewhere(
model = build_model(spec, reasoning_effort="high")
assert getattr(model, "reasoning_effort", None) is None
assert "reasoning_effort is not supported" in capsys.readouterr().err


# --- streamed usage --------------------------------------------------------------


def _merged_usage(usages: list[tuple[int, int] | None]) -> tuple[int, int]:
"""Stream chunks carrying ``usages`` through the delta rewrite and merge."""
chunks = [
ChatGenerationChunk(
message=AIMessageChunk(
content="x",
usage_metadata=None
if u is None
else {
"input_tokens": u[0],
"output_tokens": u[1],
"total_tokens": u[0] + u[1],
},
)
)
for u in usages
]
seen = None
for chunk in chunks:
seen = _usage_to_delta(chunk, seen)
merged = chunks[0]
for chunk in chunks[1:]:
merged += chunk
assert isinstance(merged.message, AIMessageChunk)
usage = merged.message.usage_metadata
assert usage is not None
return usage["input_tokens"], usage["output_tokens"]


def test_cumulative_streamed_usage_is_counted_once() -> None:
"""A backend sending the running total on every chunk must not be summed."""
assert _merged_usage([(100, 1), (100, 2), None, (100, 5)]) == (100, 5)


def test_single_final_usage_chunk_is_unchanged() -> None:
assert _merged_usage([None, None, (100, 5)]) == (100, 5)
Loading