From 6a9a200bfb4835b1f7272a801580d35217dc797c Mon Sep 17 00:00:00 2001 From: Nirmal T Date: Thu, 8 Oct 2026 15:32:02 +0530 Subject: [PATCH] fix(gemini): migrate to google.golang.org/genai SDK The provider used github.com/google/generative-ai-go, which Google deprecated and which cannot parse Gemini 3.x responses: every stream ended in "invalid character ']'" (turning complete answers into errors), and function-calling turns came back as an unknown finish reason with no text. - Rewrite the Gemini provider on google.golang.org/genai. - Carry Gemini 3 thought signatures on ToolCall.ThoughtSignature and replay them, so multi-step tool loops are accepted. - Give each function call a unique ID and Index (parallel calls to the same tool used to collide on the function name). - Without declared tools, replay tool history as text (Gemini rejects function-call history when no tools are declared). - Report usage (thinking tokens count as completion), pass tool schemas through as raw JSON Schema, surface the API's real error and status, skip thought parts, honour BaseURL and CustomHeaders. Co-Authored-By: Claude Opus 5.5 --- go.mod | 16 +- go.sum | 31 +-- providers/gemini/converter.go | 432 ++++++++++++++++-------------- providers/gemini/doc.go | 14 +- providers/gemini/provider.go | 237 +++++++--------- providers/gemini/provider_test.go | 362 +++++++++++++++++++++++++ types.go | 4 + 7 files changed, 707 insertions(+), 389 deletions(-) create mode 100644 providers/gemini/provider_test.go diff --git a/go.mod b/go.mod index d28f34a..47a5571 100644 --- a/go.mod +++ b/go.mod @@ -4,28 +4,25 @@ go 1.26.0 require ( github.com/anthropics/anthropic-sdk-go v1.78.0 - github.com/google/generative-ai-go v0.20.1 github.com/openai/openai-go v1.12.0 - google.golang.org/api v0.297.0 + google.golang.org/genai v1.73.0 ) require ( - cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/ai v0.8.0 // indirect + cloud.google.com/go v0.116.0 // indirect cloud.google.com/go/auth v0.23.2 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/longrunning v0.5.7 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.20 // indirect github.com/googleapis/gax-go/v2 v2.24.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/invopop/jsonschema v0.14.0 // indirect github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect @@ -34,7 +31,6 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect @@ -42,12 +38,10 @@ require ( go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/net v0.58.0 // indirect - golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect - golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d // indirect + google.golang.org/api v0.297.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect diff --git a/go.sum b/go.sum index e8a6ac3..16866f8 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,9 @@ -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= -cloud.google.com/go/ai v0.8.0 h1:rXUEz8Wp2OlrM8r1bfmpF2+VKqc1VJpafE3HgzRnD/w= -cloud.google.com/go/ai v0.8.0/go.mod h1:t3Dfk4cM61sytiggo2UyGsDVW3RF1qGZaUKDrZFyqkE= +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= cloud.google.com/go/auth v0.23.2 h1:pxSCpfiji41hpzpPdMCftEUCezpgpqmmDdYiAjCKXxo= cloud.google.com/go/auth v0.23.2/go.mod h1:4DhBRcqvtljQN3dJ57qtqbib5ZGCYE5f2crfiiC2EM0= -cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= -cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= -cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= github.com/anthropics/anthropic-sdk-go v1.78.0 h1:8uluL7g84HtSSm347tMGtrUuCEBLmme7vOcFCMn7bnk= github.com/anthropics/anthropic-sdk-go v1.78.0/go.mod h1:x+lPk/cCl48uRegeP0hlYYBN1b7bEBTveInIMgLicnY= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= @@ -18,17 +12,10 @@ github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJ github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= -github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= -github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= -github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= -github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= -github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -38,8 +25,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/generative-ai-go v0.20.1 h1:6dEIujpgN2V0PgLhr6c/M1ynRdc7ARtiIDPFzj45uNQ= -github.com/google/generative-ai-go v0.20.1/go.mod h1:TjOnZJmZKzarWbjUJgy+r3Ee7HGBRVLhOIgupnwR4Bg= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= @@ -50,14 +35,14 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.20 h1:t/xL64VUoN69MuMRQu github.com/googleapis/enterprise-certificate-proxy v0.3.20/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= github.com/googleapis/gax-go/v2 v2.24.0 h1:myMaPYyF9MecEmvQqMqomIwn9t/4KCZN9qnwsS76wlg= github.com/googleapis/gax-go/v2 v2.24.0/go.mod h1:IaTHBDd7NHxSCiu0vEs8pQZu4dGZrWwuSoxCnk16OFM= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0= github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y= github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY= github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= -github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 h1:uOfcYT+3QungH6tIGSVCR/Y3KJmgJiHcojJbMTPDZAI= @@ -76,8 +61,6 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= @@ -96,20 +79,18 @@ golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= -golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= -golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= -golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.297.0 h1:WktxTsnnx0yZNnsR6j0q6hR21RnnK81FHTOPy/ux4OE= google.golang.org/api v0.297.0/go.mod h1:S4m8x0M6OkQpkOzGk1y9JG2sm4fFQrMh6dxzjCTszhE= +google.golang.org/genai v1.73.0 h1:QiB89pqDl5xm3fUcY2QTvyQ2kQzSeDuKd3E7vyod8BI= +google.golang.org/genai v1.73.0/go.mod h1:2j40fGpqPPIZNjaDKmjGGjYwCJi6Nm2ofmyP65GnqtY= google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d h1:C9v1o0/4quuhOAfmRXA2j+we0PqZIp8traLdeogF3Ms= google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d/go.mod h1:Wz2wFJntZFmLGo7pLDXZ3wYk5hyc0Mb+SkHhDDXT+lU= google.golang.org/genproto/googleapis/api v0.0.0-20260715232425-e75dac1f907d h1:QwnJwPte4XXAkhPu26LTDIahnsMSUV0kK8HkxbC+Pc4= diff --git a/providers/gemini/converter.go b/providers/gemini/converter.go index c54cfde..16e204d 100644 --- a/providers/gemini/converter.go +++ b/providers/gemini/converter.go @@ -3,286 +3,304 @@ package gemini import ( "encoding/base64" "encoding/json" - "time" + "errors" + "fmt" + "strings" llmrouter "github.com/bluefunda/llmrouter" - "github.com/google/generative-ai-go/genai" + "google.golang.org/genai" ) -// convertHistory converts llmrouter messages to Gemini chat history -// Returns the history and the last user message parts (which should be sent separately) -func convertHistory(msgs []llmrouter.Message) ([]*genai.Content, []genai.Part) { - var history []*genai.Content - var lastUserParts []genai.Part +// convertMessages converts llmrouter messages to Gemini contents plus the system instruction. +// +// toolsDeclared reports whether the request declares tools. Gemini rejects function-call and +// function-response history when no tools are declared, so without tools (e.g. a tool-free +// retry after a rejected tool call) earlier tool exchanges are replayed as plain text instead. +func convertMessages(msgs []llmrouter.Message, toolsDeclared bool) ([]*genai.Content, *genai.Content) { + var contents []*genai.Content + var system []string + toolNames := map[string]string{} // tool-call ID → function name, for results missing Name - for i, msg := range msgs { + for _, msg := range msgs { switch msg.Role { case llmrouter.RoleSystem: - // System messages are handled separately via SystemInstruction - continue + if strings.TrimSpace(msg.Content) != "" { + system = append(system, msg.Content) + } case llmrouter.RoleUser: - parts := buildUserParts(msg) - // If this is the last message, save it for sending - if i == len(msgs)-1 { - lastUserParts = parts - continue + if parts := buildUserParts(msg); len(parts) > 0 { + contents = append(contents, &genai.Content{Role: genai.RoleUser, Parts: parts}) } - history = append(history, &genai.Content{ - Role: "user", - Parts: parts, - }) case llmrouter.RoleAssistant: - parts := []genai.Part{} + var parts []*genai.Part if msg.Content != "" { - parts = append(parts, genai.Text(msg.Content)) + parts = append(parts, &genai.Part{Text: msg.Content}) } for _, tc := range msg.ToolCalls { - var args map[string]interface{} - _ = json.Unmarshal([]byte(tc.Function.Arguments), &args) - parts = append(parts, genai.FunctionCall{ - Name: tc.Function.Name, - Args: args, - }) + toolNames[tc.ID] = tc.Function.Name + if toolsDeclared { + parts = append(parts, &genai.Part{ + FunctionCall: &genai.FunctionCall{ + Name: tc.Function.Name, + Args: parseArgs(tc.Function.Arguments), + }, + // Gemini 3 rejects function-call history without the signature it issued. + ThoughtSignature: tc.ThoughtSignature, + }) + } else { + parts = append(parts, &genai.Part{ + Text: fmt.Sprintf("[Called tool %s with arguments %s]", tc.Function.Name, tc.Function.Arguments), + }) + } } if len(parts) > 0 { - history = append(history, &genai.Content{ - Role: "model", - Parts: parts, - }) + contents = append(contents, &genai.Content{Role: genai.RoleModel, Parts: parts}) } case llmrouter.RoleTool: - // Tool results - var result map[string]interface{} - _ = json.Unmarshal([]byte(msg.Content), &result) - if result == nil { - result = map[string]interface{}{"result": msg.Content} + name := msg.Name + if name == "" { + name = toolNames[msg.ToolCallID] } - history = append(history, &genai.Content{ - Role: "user", - Parts: []genai.Part{ - genai.FunctionResponse{ - Name: msg.Name, - Response: result, - }, - }, - }) + if !toolsDeclared { + contents = append(contents, &genai.Content{ + Role: genai.RoleUser, + Parts: []*genai.Part{{Text: fmt.Sprintf("[Result of tool %s]\n%s", name, msg.Content)}}, + }) + continue + } + part := &genai.Part{FunctionResponse: &genai.FunctionResponse{ + Name: name, + Response: toolResult(msg.Content), + }} + // Results of parallel calls go back together in one user turn. + if last := lastContent(contents); last != nil && isFunctionResponseTurn(last) { + last.Parts = append(last.Parts, part) + continue + } + contents = append(contents, &genai.Content{Role: genai.RoleUser, Parts: []*genai.Part{part}}) } } - return history, lastUserParts + var instruction *genai.Content + if len(system) > 0 { + instruction = &genai.Content{Parts: []*genai.Part{{Text: strings.Join(system, "\n\n")}}} + } + return contents, instruction } -// buildUserParts converts a user message (text-only or multimodal) to Gemini parts -func buildUserParts(msg llmrouter.Message) []genai.Part { - if len(msg.ContentParts) > 0 { - parts := make([]genai.Part, 0, len(msg.ContentParts)) - for _, p := range msg.ContentParts { - switch p.Type { - case "text": - parts = append(parts, genai.Text(p.Text)) - case "image_url": - if p.ImageURL != nil && p.ImageURL.Base64 != "" { - imgBytes, err := base64.StdEncoding.DecodeString(p.ImageURL.Base64) - if err == nil { - parts = append(parts, genai.ImageData(p.ImageURL.MediaType, imgBytes)) - } - } - } - } - return parts +func lastContent(contents []*genai.Content) *genai.Content { + if len(contents) == 0 { + return nil } - return []genai.Part{genai.Text(msg.Content)} + return contents[len(contents)-1] } -// convertTools converts llmrouter tools to Gemini format -func convertTools(tools []llmrouter.Tool) []*genai.Tool { - funcDecls := make([]*genai.FunctionDeclaration, len(tools)) - - for i, tool := range tools { - var schema *genai.Schema - if tool.Function.Parameters != nil { - var params map[string]interface{} - _ = json.Unmarshal(tool.Function.Parameters, ¶ms) - schema = convertSchema(params) - } - - funcDecls[i] = &genai.FunctionDeclaration{ - Name: tool.Function.Name, - Description: tool.Function.Description, - Parameters: schema, +func isFunctionResponseTurn(c *genai.Content) bool { + if c.Role != genai.RoleUser || len(c.Parts) == 0 { + return false + } + for _, p := range c.Parts { + if p.FunctionResponse == nil { + return false } } + return true +} - return []*genai.Tool{ - {FunctionDeclarations: funcDecls}, +// parseArgs decodes a tool call's JSON arguments; malformed or empty arguments become {}. +func parseArgs(arguments string) map[string]any { + args := map[string]any{} + if strings.TrimSpace(arguments) != "" { + _ = json.Unmarshal([]byte(arguments), &args) } + return args } -// convertSchema converts a JSON schema to Gemini Schema -func convertSchema(params map[string]interface{}) *genai.Schema { - if params == nil { - return nil +// toolResult wraps a tool's output as a Gemini function response: a JSON object is passed +// through, anything else goes under "output". +func toolResult(content string) map[string]any { + var obj map[string]any + if err := json.Unmarshal([]byte(content), &obj); err == nil && obj != nil { + return obj } + return map[string]any{"output": content} +} - schema := &genai.Schema{ - Type: genai.TypeObject, +// buildUserParts converts a user message (text-only or multimodal) to Gemini parts. +func buildUserParts(msg llmrouter.Message) []*genai.Part { + if len(msg.ContentParts) == 0 { + if msg.Content == "" { + return nil + } + return []*genai.Part{{Text: msg.Content}} } - if props, ok := params["properties"].(map[string]interface{}); ok { - schema.Properties = make(map[string]*genai.Schema) - for name, prop := range props { - if propMap, ok := prop.(map[string]interface{}); ok { - schema.Properties[name] = convertPropertySchema(propMap) + parts := make([]*genai.Part, 0, len(msg.ContentParts)) + for _, p := range msg.ContentParts { + switch p.Type { + case "text": + if p.Text != "" { + parts = append(parts, &genai.Part{Text: p.Text}) + } + case "image_url": + if p.ImageURL != nil && p.ImageURL.Base64 != "" { + if data, err := base64.StdEncoding.DecodeString(p.ImageURL.Base64); err == nil { + parts = append(parts, &genai.Part{InlineData: &genai.Blob{MIMEType: p.ImageURL.MediaType, Data: data}}) + } } } } + return parts +} - if required, ok := params["required"].([]interface{}); ok { - for _, r := range required { - if s, ok := r.(string); ok { - schema.Required = append(schema.Required, s) +// convertTools converts llmrouter tools to Gemini function declarations. Parameters are passed +// as raw JSON Schema (ParametersJsonSchema), so nothing in the schema is lost in conversion. +func convertTools(tools []llmrouter.Tool) []*genai.Tool { + decls := make([]*genai.FunctionDeclaration, 0, len(tools)) + for _, tool := range tools { + decl := &genai.FunctionDeclaration{ + Name: tool.Function.Name, + Description: tool.Function.Description, + } + if len(tool.Function.Parameters) > 0 { + var schema any + if err := json.Unmarshal(tool.Function.Parameters, &schema); err == nil && schema != nil { + decl.ParametersJsonSchema = schema } } + decls = append(decls, decl) } + return []*genai.Tool{{FunctionDeclarations: decls}} +} - return schema +// accumulator turns Gemini response chunks into llmrouter events and the final response. +type accumulator struct { + content strings.Builder + toolCalls []llmrouter.ToolCall + finish genai.FinishReason + meta *genai.GenerateContentResponseUsageMetadata } -func convertPropertySchema(prop map[string]interface{}) *genai.Schema { - schema := &genai.Schema{} +func newAccumulator() *accumulator { return &accumulator{} } - if t, ok := prop["type"].(string); ok { - switch t { - case "string": - schema.Type = genai.TypeString - case "number": - schema.Type = genai.TypeNumber - case "integer": - schema.Type = genai.TypeInteger - case "boolean": - schema.Type = genai.TypeBoolean - case "array": - schema.Type = genai.TypeArray - if items, ok := prop["items"].(map[string]interface{}); ok { - schema.Items = convertPropertySchema(items) - } - case "object": - schema.Type = genai.TypeObject - if props, ok := prop["properties"].(map[string]interface{}); ok { - schema.Properties = make(map[string]*genai.Schema) - for name, p := range props { - if pm, ok := p.(map[string]interface{}); ok { - schema.Properties[name] = convertPropertySchema(pm) - } - } +// add processes one response chunk, emitting events for text and tool calls. +func (a *accumulator) add(resp *genai.GenerateContentResponse, emit func(llmrouter.Event)) error { + if resp == nil { + return nil + } + if resp.UsageMetadata != nil { + a.meta = resp.UsageMetadata + } + if len(resp.Candidates) == 0 { + if fb := resp.PromptFeedback; fb != nil && fb.BlockReason != "" { + return &llmrouter.APIError{ + Provider: "gemini", + Type: "prompt_blocked", + Message: fmt.Sprintf("prompt blocked: %s %s", fb.BlockReason, fb.BlockReasonMessage), } } + return nil } - if desc, ok := prop["description"].(string); ok { - schema.Description = desc + cand := resp.Candidates[0] + if cand.FinishReason != "" { + a.finish = cand.FinishReason } - - if enum, ok := prop["enum"].([]interface{}); ok { - for _, e := range enum { - if s, ok := e.(string); ok { - schema.Enum = append(schema.Enum, s) - } + if cand.Content == nil { + return nil + } + for _, part := range cand.Content.Parts { + switch { + case part == nil: + case part.FunctionCall != nil: + tc := a.addToolCall(part) + emit(llmrouter.Event{ + Type: llmrouter.EventToolCallDelta, + Delta: &llmrouter.Delta{ToolCalls: []llmrouter.ToolCall{tc}}, + }) + case part.Thought: + // Thought summaries are not requested; skip any so they never leak into the answer. + case part.Text != "": + a.content.WriteString(part.Text) + emit(llmrouter.Event{Type: llmrouter.EventContentDelta, Content: part.Text}) } } - - return schema + return nil } -// convertResponse converts Gemini response to OpenAI-compatible format -func convertResponse(resp *genai.GenerateContentResponse, model, provider string) *llmrouter.Response { - var content string - var toolCalls []llmrouter.ToolCall - - if len(resp.Candidates) > 0 && resp.Candidates[0].Content != nil { - for _, part := range resp.Candidates[0].Content.Parts { - switch p := part.(type) { - case genai.Text: - content += string(p) - case genai.FunctionCall: - args, _ := convertFunctionCallArgs(p.Args) - toolCalls = append(toolCalls, llmrouter.ToolCall{ - ID: p.Name, - Type: "function", - Function: llmrouter.FuncCall{ - Name: p.Name, - Arguments: args, - }, - }) - } - } +// addToolCall records a function call. Each call gets a unique ID and Index even when Gemini +// omits an ID: the old provider used the function name as the ID, so two parallel calls to the +// same tool collided and were merged into one. +func (a *accumulator) addToolCall(part *genai.Part) llmrouter.ToolCall { + fc := part.FunctionCall + index := len(a.toolCalls) + id := fc.ID + if id == "" { + id = fmt.Sprintf("call_%d_%s", index, fc.Name) } - - finishReason := "stop" - if len(toolCalls) > 0 { - finishReason = "tool_calls" - } else if len(resp.Candidates) > 0 { - switch resp.Candidates[0].FinishReason { - case genai.FinishReasonMaxTokens: - finishReason = "length" - case genai.FinishReasonStop: - finishReason = "stop" - case genai.FinishReasonSafety: - finishReason = "content_filter" + args := "{}" + if len(fc.Args) > 0 { + if b, err := json.Marshal(fc.Args); err == nil { + args = string(b) } } - - var usage *llmrouter.Usage - if resp.UsageMetadata != nil { - usage = &llmrouter.Usage{ - PromptTokens: int(resp.UsageMetadata.PromptTokenCount), - CompletionTokens: int(resp.UsageMetadata.CandidatesTokenCount), - TotalTokens: int(resp.UsageMetadata.TotalTokenCount), - CachedPromptTokens: int(resp.UsageMetadata.CachedContentTokenCount), - } + tc := llmrouter.ToolCall{ + ID: id, + Type: "function", + Function: llmrouter.FuncCall{Name: fc.Name, Arguments: args}, + Index: &index, + ThoughtSignature: part.ThoughtSignature, } + a.toolCalls = append(a.toolCalls, tc) + return tc +} - return &llmrouter.Response{ - Model: model, - Provider: provider, - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []llmrouter.Choice{ - { - Index: 0, - Message: &llmrouter.Message{ - Role: llmrouter.RoleAssistant, - Content: content, - ToolCalls: toolCalls, - }, - FinishReason: finishReason, - }, - }, - Usage: usage, +// finishReason maps Gemini's finish reason to the OpenAI-compatible value. +func (a *accumulator) finishReason() string { + if len(a.toolCalls) > 0 { + return "tool_calls" + } + switch a.finish { + case "", genai.FinishReasonStop: + return "stop" + case genai.FinishReasonMaxTokens: + return "length" + case genai.FinishReasonSafety, genai.FinishReasonRecitation, genai.FinishReasonBlocklist, + genai.FinishReasonProhibitedContent, genai.FinishReasonSPII, genai.FinishReasonImageSafety: + return "content_filter" + default: + // e.g. MALFORMED_FUNCTION_CALL — surfaced as-is so callers can see what happened. + return strings.ToLower(string(a.finish)) } } -// convertFunctionCallArgs converts function call args to JSON string -func convertFunctionCallArgs(args map[string]interface{}) (string, error) { - if args == nil { - return "{}", nil +// usage reports token usage; thinking tokens count as completion tokens (they are billed so). +func (a *accumulator) usage() *llmrouter.Usage { + if a.meta == nil { + return nil + } + return &llmrouter.Usage{ + PromptTokens: int(a.meta.PromptTokenCount), + CompletionTokens: int(a.meta.CandidatesTokenCount + a.meta.ThoughtsTokenCount), + TotalTokens: int(a.meta.TotalTokenCount), + CachedPromptTokens: int(a.meta.CachedContentTokenCount), } - b, err := json.Marshal(args) - return string(b), err } -// wrapError wraps Gemini errors +// wrapError wraps Gemini errors, keeping the HTTP status when the SDK reports one. func wrapError(err error) error { if err == nil { return nil } - - return &llmrouter.APIError{ - Provider: "gemini", - Message: err.Error(), - Err: err, + apiErr := &llmrouter.APIError{Provider: "gemini", Message: err.Error(), Err: err} + var gErr genai.APIError + if errors.As(err, &gErr) { + apiErr.StatusCode = gErr.Code + apiErr.Type = gErr.Status + apiErr.Message = gErr.Message } + return apiErr } diff --git a/providers/gemini/doc.go b/providers/gemini/doc.go index 47ae03d..069b174 100644 --- a/providers/gemini/doc.go +++ b/providers/gemini/doc.go @@ -1,15 +1,15 @@ // Package gemini implements the llmrouter.Provider interface for Google -// Gemini models using the official Google Generative AI Go SDK. +// Gemini models using the official Google Gen AI Go SDK (google.golang.org/genai). // -// Gemini requires a context during initialisation: +// Create a provider from the GEMINI_API_KEY environment variable: // -// p, err := gemini.NewFromEnv(ctx) // reads GEMINI_API_KEY +// p, err := gemini.NewFromEnv() // // Or with an explicit key: // -// p, err := gemini.New(ctx, gemini.Config{APIKey: "..."}) +// p, err := gemini.New(llmrouter.ProviderConfig{APIKey: "..."}) // -// Supported models include gemini-2.0-flash, gemini-1.5-pro, and -// gemini-1.5-flash. The provider supports streaming, tool calling, -// and multimodal inputs (text + images + documents). +// The provider supports streaming, tool calling (including Gemini 3 thought +// signatures, carried on llmrouter.ToolCall.ThoughtSignature), and text and +// image inputs. package gemini diff --git a/providers/gemini/provider.go b/providers/gemini/provider.go index ddca047..7af3f94 100644 --- a/providers/gemini/provider.go +++ b/providers/gemini/provider.go @@ -2,16 +2,19 @@ package gemini import ( "context" + "net/http" "os" "time" llmrouter "github.com/bluefunda/llmrouter" - "github.com/google/generative-ai-go/genai" - "google.golang.org/api/iterator" - "google.golang.org/api/option" + "google.golang.org/genai" ) -// Provider handles Google Gemini API +// Provider handles the Google Gemini API through the official google.golang.org/genai SDK. +// +// It replaced github.com/google/generative-ai-go, which Google deprecated and which can no +// longer parse Gemini 3.x streams: every stream ended in "invalid character ']'", and +// function-calling turns came back as an unknown finish reason with no content. type Provider struct { client *genai.Client model string @@ -20,19 +23,18 @@ type Provider struct { // DefaultModels is the list of available Gemini models var DefaultModels = []string{ - "gemini-1.5-pro", - "gemini-1.5-flash", - "gemini-2.0-flash-exp", - "gemini-1.0-pro", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", } // New creates a new Gemini provider. -// The Gemini SDK requires a context for client construction; context.Background() -// is used internally so the provider lifetime is not tied to a caller's context. +// The SDK takes a context for client construction; context.Background() is used so the +// provider lifetime is not tied to a caller's context. cfg.BaseURL overrides the API +// endpoint (used by tests); cfg.CustomHeaders are sent on every request. func New(cfg llmrouter.ProviderConfig) (*Provider, error) { model := cfg.Model if model == "" { - model = "gemini-1.5-flash" + model = "gemini-3.5-flash-lite" } models := cfg.Models @@ -40,12 +42,21 @@ func New(cfg llmrouter.ProviderConfig) (*Provider, error) { models = DefaultModels } - opts := []option.ClientOption{} - if cfg.APIKey != "" { - opts = append(opts, option.WithAPIKey(cfg.APIKey)) + cc := &genai.ClientConfig{ + APIKey: cfg.APIKey, + Backend: genai.BackendGeminiAPI, + } + if cfg.BaseURL != "" { + cc.HTTPOptions.BaseURL = cfg.BaseURL + } + if len(cfg.CustomHeaders) > 0 { + cc.HTTPOptions.Headers = http.Header{} + for key, value := range cfg.CustomHeaders { + cc.HTTPOptions.Headers.Set(key, value) + } } - client, err := genai.NewClient(context.Background(), opts...) + client, err := genai.NewClient(context.Background(), cc) if err != nil { return nil, err } @@ -64,9 +75,9 @@ func NewFromEnv() (*Provider, error) { }) } -// Close closes the Gemini client +// Close releases the provider. The genai client holds no resources that need closing. func (p *Provider) Close() error { - return p.client.Close() + return nil } func (p *Provider) Name() string { @@ -79,47 +90,32 @@ func (p *Provider) Models() []string { return out } -func (p *Provider) Complete(ctx context.Context, req *llmrouter.Request) (*llmrouter.Response, error) { - modelName := req.Model - if modelName == "" { - modelName = p.model - } - - model := p.client.GenerativeModel(modelName) - configureModel(model, req) - - if len(req.Tools) > 0 { - model.Tools = convertTools(req.Tools) +func (p *Provider) modelName(req *llmrouter.Request) string { + if req.Model != "" { + return req.Model } + return p.model +} - chat := model.StartChat() - history, lastParts := convertHistory(req.Messages) - chat.History = history +func (p *Provider) Complete(ctx context.Context, req *llmrouter.Request) (*llmrouter.Response, error) { + modelName := p.modelName(req) + contents, config := buildRequest(req) - resp, err := chat.SendMessage(ctx, lastParts...) + resp, err := p.client.Models.GenerateContent(ctx, modelName, contents, config) if err != nil { return nil, wrapError(err) } - return convertResponse(resp, modelName, p.Name()), nil + acc := newAccumulator() + if err := acc.add(resp, func(llmrouter.Event) {}); err != nil { + return nil, err + } + return acc.response(modelName, p.Name()), nil } func (p *Provider) Stream(ctx context.Context, req *llmrouter.Request) (*llmrouter.StreamResult, error) { - modelName := req.Model - if modelName == "" { - modelName = p.model - } - - model := p.client.GenerativeModel(modelName) - configureModel(model, req) - - if len(req.Tools) > 0 { - model.Tools = convertTools(req.Tools) - } - - chat := model.StartChat() - history, lastParts := convertHistory(req.Messages) - chat.History = history + modelName := p.modelName(req) + contents, config := buildRequest(req) ctx, cancel := context.WithCancel(ctx) ch := make(chan llmrouter.Event) @@ -130,115 +126,78 @@ func (p *Provider) Stream(ctx context.Context, req *llmrouter.Request) (*llmrout defer close(ch) defer cancel() - iter := chat.SendMessageStream(ctx, lastParts...) - - var fullContent string - var toolCalls []llmrouter.ToolCall - - for { - resp, err := iter.Next() - if err == iterator.Done { - break + emit := func(ev llmrouter.Event) { + select { + case ch <- ev: + case <-ctx.Done(): } + } + + acc := newAccumulator() + for resp, err := range p.client.Models.GenerateContentStream(ctx, modelName, contents, config) { if err != nil { - ch <- llmrouter.Event{ - Type: llmrouter.EventError, - Error: wrapError(err), - } + emit(llmrouter.Event{Type: llmrouter.EventError, Error: wrapError(err)}) return } - - for _, candidate := range resp.Candidates { - if candidate.Content == nil { - continue - } - for _, part := range candidate.Content.Parts { - switch p := part.(type) { - case genai.Text: - content := string(p) - fullContent += content - ch <- llmrouter.Event{ - Type: llmrouter.EventContentDelta, - Content: content, - } - case genai.FunctionCall: - args, _ := convertFunctionCallArgs(p.Args) - tc := llmrouter.ToolCall{ - ID: p.Name, - Type: "function", - Function: llmrouter.FuncCall{ - Name: p.Name, - Arguments: args, - }, - } - toolCalls = append(toolCalls, tc) - ch <- llmrouter.Event{ - Type: llmrouter.EventToolCallDelta, - Delta: &llmrouter.Delta{ - ToolCalls: []llmrouter.ToolCall{tc}, - }, - } - } - } + if err := acc.add(resp, emit); err != nil { + emit(llmrouter.Event{Type: llmrouter.EventError, Error: err}) + return } } - - finishReason := "stop" - if len(toolCalls) > 0 { - finishReason = "tool_calls" - } - - ch <- llmrouter.Event{ - Type: llmrouter.EventDone, - Response: &llmrouter.Response{ - Model: modelName, - Provider: p.Name(), - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []llmrouter.Choice{ - { - Index: 0, - Message: &llmrouter.Message{ - Role: llmrouter.RoleAssistant, - Content: fullContent, - ToolCalls: toolCalls, - }, - FinishReason: finishReason, - }, - }, - }, - } + emit(llmrouter.Event{ + Type: llmrouter.EventDone, + Response: acc.response(modelName, p.Name()), + }) }() return res, nil } -func configureModel(model *genai.GenerativeModel, req *llmrouter.Request) { - if req.Temperature != nil { - temp := float32(*req.Temperature) - model.Temperature = &temp +// buildRequest converts an llmrouter request into Gemini contents and generation config. +func buildRequest(req *llmrouter.Request) ([]*genai.Content, *genai.GenerateContentConfig) { + toolsDeclared := len(req.Tools) > 0 + contents, system := convertMessages(req.Messages, toolsDeclared) + + config := &genai.GenerateContentConfig{ + SystemInstruction: system, + MaxOutputTokens: 16384, + StopSequences: req.Stop, } if req.MaxTokens != nil { - tokens := int32(*req.MaxTokens) - model.MaxOutputTokens = &tokens - } else { - tokens := int32(16384) - model.MaxOutputTokens = &tokens + config.MaxOutputTokens = int32(*req.MaxTokens) + } + if req.Temperature != nil { + temp := float32(*req.Temperature) + config.Temperature = &temp } if req.TopP != nil { topP := float32(*req.TopP) - model.TopP = &topP + config.TopP = &topP } - if len(req.Stop) > 0 { - model.StopSequences = req.Stop + if toolsDeclared { + config.Tools = convertTools(req.Tools) } + return contents, config +} - for _, msg := range req.Messages { - if msg.Role == llmrouter.RoleSystem { - model.SystemInstruction = &genai.Content{ - Parts: []genai.Part{genai.Text(msg.Content)}, - } - break - } +// response builds the final OpenAI-compatible response once the stream ends. +func (a *accumulator) response(model, provider string) *llmrouter.Response { + return &llmrouter.Response{ + Model: model, + Provider: provider, + Object: "chat.completion", + Created: time.Now().Unix(), + Choices: []llmrouter.Choice{ + { + Index: 0, + Message: &llmrouter.Message{ + Role: llmrouter.RoleAssistant, + Content: a.content.String(), + ToolCalls: a.toolCalls, + }, + FinishReason: a.finishReason(), + }, + }, + Usage: a.usage(), } } diff --git a/providers/gemini/provider_test.go b/providers/gemini/provider_test.go new file mode 100644 index 0000000..aaf30ad --- /dev/null +++ b/providers/gemini/provider_test.go @@ -0,0 +1,362 @@ +package gemini + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + llmrouter "github.com/bluefunda/llmrouter" + "google.golang.org/genai" +) + +// fakeGemini serves canned Gemini API responses and records each request body. +type fakeGemini struct { + server *httptest.Server + bodies []map[string]any +} + +// newFakeGemini streams chunks as server-sent events for streamGenerateContent, and +// answers generateContent with the last chunk as one JSON body. status != 200 returns errBody. +func newFakeGemini(t *testing.T, status int, errBody string, chunks ...string) *fakeGemini { + t.Helper() + f := &fakeGemini{} + f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + var body map[string]any + _ = json.Unmarshal(raw, &body) + f.bodies = append(f.bodies, body) + + if status != http.StatusOK { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, errBody) + return + } + if strings.Contains(r.URL.Path, ":streamGenerateContent") { + w.Header().Set("Content-Type", "text/event-stream") + for _, c := range chunks { + _, _ = fmt.Fprintf(w, "data: %s\r\n\r\n", c) + } + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, chunks[len(chunks)-1]) + })) + t.Cleanup(f.server.Close) + return f +} + +func (f *fakeGemini) provider(t *testing.T) *Provider { + t.Helper() + p, err := New(llmrouter.ProviderConfig{APIKey: "test-key", BaseURL: f.server.URL}) + if err != nil { + t.Fatalf("New: %v", err) + } + return p +} + +func collect(t *testing.T, p *Provider, req *llmrouter.Request) []llmrouter.Event { + t.Helper() + stream, err := p.Stream(context.Background(), req) + if err != nil { + t.Fatalf("Stream: %v", err) + } + defer func() { _ = stream.Close() }() + var events []llmrouter.Event + for stream.Next() { + events = append(events, stream.Event()) + } + return events +} + +func userRequest(text string) *llmrouter.Request { + return &llmrouter.Request{ + Model: "gemini-3.5-flash-lite", + Messages: []llmrouter.Message{{Role: llmrouter.RoleUser, Content: text}}, + } +} + +// The deprecated SDK failed at the end of every Gemini 3 stream ("invalid character ']'"), +// which turned complete answers into errors. A normal stream must end in EventDone only. +func TestStream_TextAnswerEndsCleanly(t *testing.T) { + f := newFakeGemini(t, http.StatusOK, "", + `{"candidates":[{"content":{"role":"model","parts":[{"text":"Hello, "}]}}]}`, + `{"candidates":[{"content":{"role":"model","parts":[{"text":"world."}]},"finishReason":"STOP"}],`+ + `"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":4,"thoughtsTokenCount":6,"totalTokenCount":20,"cachedContentTokenCount":3}}`, + ) + + events := collect(t, f.provider(t), userRequest("Say hello")) + + var text strings.Builder + var done *llmrouter.Response + for _, ev := range events { + switch ev.Type { + case llmrouter.EventError: + t.Fatalf("unexpected error event: %v", ev.Error) + case llmrouter.EventContentDelta: + text.WriteString(ev.Content) + case llmrouter.EventDone: + done = ev.Response + } + } + if text.String() != "Hello, world." { + t.Fatalf("streamed text = %q", text.String()) + } + if done == nil { + t.Fatal("no EventDone") + } + if got := done.Choices[0].FinishReason; got != "stop" { + t.Errorf("finish reason = %q, want stop", got) + } + if got := done.Choices[0].Message.Content; got != "Hello, world." { + t.Errorf("final content = %q", got) + } + u := done.Usage + if u == nil || u.PromptTokens != 10 || u.CompletionTokens != 10 || u.TotalTokens != 20 || u.CachedPromptTokens != 3 { + t.Errorf("usage = %+v, want prompt 10, completion 10 (4 answer + 6 thinking), total 20, cached 3", u) + } +} + +// Gemini 3 issues a thought signature with a function call and rejects the next request if it is +// not sent back. Two parallel calls to the same tool must also stay two distinct calls: the old +// provider used the function name as the ID, so they collided (#294 in cai-llm-router). +func TestStream_FunctionCallsKeepSignatureAndUniqueIDs(t *testing.T) { + f := newFakeGemini(t, http.StatusOK, "", + `{"candidates":[{"content":{"role":"model","parts":[`+ + `{"functionCall":{"name":"fetch","args":{"url":"https://a.example"}},"thoughtSignature":"c2lnLWE="},`+ + `{"functionCall":{"name":"fetch","args":{"url":"https://b.example"}}}`+ + `]},"finishReason":"STOP"}]}`, + ) + + events := collect(t, f.provider(t), userRequest("Fetch both")) + + var deltas []llmrouter.ToolCall + var done *llmrouter.Response + for _, ev := range events { + switch ev.Type { + case llmrouter.EventError: + t.Fatalf("unexpected error event: %v", ev.Error) + case llmrouter.EventToolCallDelta: + deltas = append(deltas, ev.Delta.ToolCalls...) + case llmrouter.EventDone: + done = ev.Response + } + } + if len(deltas) != 2 { + t.Fatalf("got %d tool-call deltas, want 2", len(deltas)) + } + if deltas[0].ID == deltas[1].ID { + t.Errorf("parallel calls share ID %q", deltas[0].ID) + } + if *deltas[0].Index != 0 || *deltas[1].Index != 1 { + t.Errorf("indexes = %d, %d; want 0, 1", *deltas[0].Index, *deltas[1].Index) + } + if string(deltas[0].ThoughtSignature) != "sig-a" { + t.Errorf("first call signature = %q, want sig-a", deltas[0].ThoughtSignature) + } + if deltas[1].Function.Arguments != `{"url":"https://b.example"}` { + t.Errorf("second call args = %s", deltas[1].Function.Arguments) + } + if done == nil || done.Choices[0].FinishReason != "tool_calls" || len(done.Choices[0].Message.ToolCalls) != 2 { + t.Fatalf("EventDone = %+v, want finish tool_calls with 2 calls", done) + } +} + +// Thought parts must never be mixed into the answer text. +func TestStream_ThoughtPartsAreNotAnswerText(t *testing.T) { + f := newFakeGemini(t, http.StatusOK, "", + `{"candidates":[{"content":{"role":"model","parts":[{"text":"Let me think about this.","thought":true},{"text":"The answer is 42."}]},"finishReason":"STOP"}]}`, + ) + + events := collect(t, f.provider(t), userRequest("What is the answer?")) + + for _, ev := range events { + if ev.Type == llmrouter.EventContentDelta && strings.Contains(ev.Content, "think") { + t.Fatalf("thought leaked into answer: %q", ev.Content) + } + if ev.Type == llmrouter.EventDone && ev.Response.Choices[0].Message.Content != "The answer is 42." { + t.Fatalf("final content = %q", ev.Response.Choices[0].Message.Content) + } + } +} + +// A tool-loop follow-up: the request on the wire must replay the signature, group parallel +// results in one user turn, name a result whose message lacks Name, and pass the tool's JSON +// schema through untouched (the old converter dropped keywords such as "format"). +func TestStream_FollowUpRequestOnTheWire(t *testing.T) { + f := newFakeGemini(t, http.StatusOK, "", + `{"candidates":[{"content":{"role":"model","parts":[{"text":"Done."}]},"finishReason":"STOP"}]}`, + ) + idx0, idx1 := 0, 1 + req := &llmrouter.Request{ + Model: "gemini-3.5-flash-lite", + Messages: []llmrouter.Message{ + {Role: llmrouter.RoleSystem, Content: "Be brief."}, + {Role: llmrouter.RoleUser, Content: "Fetch both"}, + {Role: llmrouter.RoleAssistant, ToolCalls: []llmrouter.ToolCall{ + {ID: "call_0_fetch", Type: "function", Index: &idx0, ThoughtSignature: []byte("sig-a"), + Function: llmrouter.FuncCall{Name: "fetch", Arguments: `{"url":"https://a.example"}`}}, + {ID: "call_1_fetch", Type: "function", Index: &idx1, + Function: llmrouter.FuncCall{Name: "fetch", Arguments: `{"url":"https://b.example"}`}}, + }}, + {Role: llmrouter.RoleTool, ToolCallID: "call_0_fetch", Name: "fetch", Content: "page A"}, + {Role: llmrouter.RoleTool, ToolCallID: "call_1_fetch", Content: `{"status":"ok"}`}, + }, + Tools: []llmrouter.Tool{{Type: "function", Function: llmrouter.Function{ + Name: "fetch", + Description: "Fetch a URL", + Parameters: json.RawMessage(`{"type":"object","properties":{"url":{"type":"string","format":"uri"}},"required":["url"]}`), + }}}, + } + + for _, ev := range collect(t, f.provider(t), req) { + if ev.Type == llmrouter.EventError { + t.Fatalf("unexpected error event: %v", ev.Error) + } + } + if len(f.bodies) != 1 { + t.Fatalf("got %d requests, want 1", len(f.bodies)) + } + body := f.bodies[0] + + if got := jsonPath(body, "systemInstruction", "parts", 0, "text"); got != "Be brief." { + t.Errorf("system instruction = %v", got) + } + contents, _ := body["contents"].([]any) + if len(contents) != 3 { + t.Fatalf("got %d contents, want 3 (user, model calls, grouped results): %v", len(contents), contents) + } + if got := jsonPath(body, "contents", 1, "parts", 0, "thoughtSignature"); got != "c2lnLWE=" { + t.Errorf("replayed signature = %v, want base64 of sig-a", got) + } + if got := jsonPath(body, "contents", 1, "parts", 1, "functionCall", "args", "url"); got != "https://b.example" { + t.Errorf("second call url = %v", got) + } + results, _ := jsonPath(body, "contents", 2, "parts").([]any) + if len(results) != 2 { + t.Fatalf("parallel results not grouped in one turn: %v", contents[2]) + } + if got := jsonPath(body, "contents", 2, "parts", 0, "functionResponse", "response", "output"); got != "page A" { + t.Errorf("text result = %v, want under \"output\"", got) + } + if got := jsonPath(body, "contents", 2, "parts", 1, "functionResponse", "name"); got != "fetch" { + t.Errorf("unnamed result resolved to %v, want fetch", got) + } + if got := jsonPath(body, "contents", 2, "parts", 1, "functionResponse", "response", "status"); got != "ok" { + t.Errorf("JSON result = %v, want object passed through", got) + } + schema := jsonPath(body, "tools", 0, "functionDeclarations", 0, "parametersJsonSchema", "properties", "url", "format") + if schema != "uri" { + t.Errorf("tool schema keyword lost: format = %v", schema) + } +} + +// Without declared tools (a tool-free retry after a rejected tool call), Gemini rejects +// function-call history, so earlier tool exchanges are replayed as text. +func TestConvertMessages_WithoutToolsReplaysToolExchangesAsText(t *testing.T) { + msgs := []llmrouter.Message{ + {Role: llmrouter.RoleUser, Content: "Fetch it"}, + {Role: llmrouter.RoleAssistant, ToolCalls: []llmrouter.ToolCall{ + {ID: "call_0_fetch", Function: llmrouter.FuncCall{Name: "fetch", Arguments: `{"url":"https://a.example"}`}}, + }}, + {Role: llmrouter.RoleTool, ToolCallID: "call_0_fetch", Content: "page A"}, + } + + contents, _ := convertMessages(msgs, false) + + for _, c := range contents { + for _, p := range c.Parts { + if p.FunctionCall != nil || p.FunctionResponse != nil { + t.Fatalf("function part sent without declared tools: %+v", p) + } + } + } + if len(contents) != 3 { + t.Fatalf("got %d contents, want 3", len(contents)) + } + if !strings.Contains(contents[1].Parts[0].Text, "fetch") || !strings.Contains(contents[2].Parts[0].Text, "page A") { + t.Errorf("tool exchange not replayed as text: %q / %q", contents[1].Parts[0].Text, contents[2].Parts[0].Text) + } +} + +// Gemini's real error must reach the caller (the old SDK hid it behind a JSON parse error). +func TestStream_APIErrorIsReported(t *testing.T) { + f := newFakeGemini(t, http.StatusNotFound, + `{"error":{"code":404,"message":"models/gemini-x is not found","status":"NOT_FOUND"}}`) + + stream, err := f.provider(t).Stream(context.Background(), userRequest("hi")) + if err != nil { + t.Fatalf("Stream: %v", err) + } + defer func() { _ = stream.Close() }() + for stream.Next() { + t.Fatalf("unexpected event before the error: %+v", stream.Event()) + } + + // StreamResult surfaces an EventError through Err(), ending the stream. + var apiErr *llmrouter.APIError + if !errors.As(stream.Err(), &apiErr) || apiErr.StatusCode != http.StatusNotFound || + !strings.Contains(apiErr.Message, "not found") { + t.Fatalf("error = %#v, want gemini APIError 404 with the server message", stream.Err()) + } +} + +func TestComplete_ReturnsAnswer(t *testing.T) { + f := newFakeGemini(t, http.StatusOK, "", + `{"candidates":[{"content":{"role":"model","parts":[{"text":"Hi there."}]},"finishReason":"MAX_TOKENS"}]}`, + ) + + resp, err := f.provider(t).Complete(context.Background(), userRequest("hi")) + if err != nil { + t.Fatalf("Complete: %v", err) + } + if resp.Choices[0].Message.Content != "Hi there." || resp.Choices[0].FinishReason != "length" { + t.Fatalf("response = %+v", resp.Choices[0]) + } +} + +func TestFinishReason_Mapping(t *testing.T) { + cases := map[string]string{ + "STOP": "stop", + "": "stop", + "MAX_TOKENS": "length", + "SAFETY": "content_filter", + "MALFORMED_FUNCTION_CALL": "malformed_function_call", + } + for reason, want := range cases { + a := newAccumulator() + a.finish = genaiFinish(reason) + if got := a.finishReason(); got != want { + t.Errorf("%q → %q, want %q", reason, got, want) + } + } +} + +// jsonPath walks a decoded JSON value by object keys (string) and array indexes (int). +func jsonPath(v any, path ...any) any { + for _, step := range path { + switch key := step.(type) { + case string: + m, ok := v.(map[string]any) + if !ok { + return nil + } + v = m[key] + case int: + a, ok := v.([]any) + if !ok || key >= len(a) { + return nil + } + v = a[key] + } + } + return v +} + +func genaiFinish(s string) genai.FinishReason { return genai.FinishReason(s) } diff --git a/types.go b/types.go index 1b58dd3..a9785f6 100644 --- a/types.go +++ b/types.go @@ -242,6 +242,10 @@ type ToolCall struct { Type string `json:"type"` Function FuncCall `json:"function"` Index *int `json:"index,omitempty"` + // ThoughtSignature is Gemini's opaque signature for this call. Gemini 3 models reject a + // follow-up request whose function-call history is missing it, so callers must replay the + // ToolCall unchanged in the assistant message. Other providers leave it empty. + ThoughtSignature []byte `json:"thought_signature,omitempty"` } // FuncCall represents a function call