feat(agent-testing): force the test model in CompletionProvider and record used models

src/Infrastructure/BotSharp.Abstraction/Conversations/ISyntheticConversationProbe.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Agents.Models;
+
 namespace BotSharp.Abstraction.Conversations;
 
 /// <summary>
@@ -28,4 +30,10 @@ public interface ISyntheticConversationProbe
     /// traffic from the very limits it is meant to be held to.
     /// </summary>
     bool IsSynthetic(string conversationId);
+
+    /// <summary>
+    /// The provider/model the harness forces every chat completion in this conversation onto, or null
+    /// to leave it alone. Applied by CompletionProvider.GetChatCompletion.
+    /// </summary>
+    AgentLlmConfig? GetModelOverride(string conversationId) => null;
 }

src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs

@@ -60,7 +60,13 @@ public static IChatCompletion GetChatCompletion(
         AgentLlmConfig? agentConfig = null)
     {
         var completions = services.GetServices<IChatCompletion>();
-        (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, 
+        var over = GetModelOverride(services);
+        if (over != null)
+        {
+            (provider, model) = (over.Provider, over.Model);
+        }
+
+        (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId,
             multiModal: multiModal,
             agentConfig: agentConfig);
 
@@ -238,6 +244,24 @@ public static ILiveCompletion GetLiveCompletion(
         return completer;
     }
 
+    /// <summary>
+    /// A harness can force every chat completion in its conversation onto one model. Applied here, not
+    /// on the agent, so it holds no matter where the caller got its provider/model from -- an agent,
+    /// a template's llm_config, a hard-coded replacement model, or conversation state.
+    /// </summary>
+    private static AgentLlmConfig? GetModelOverride(IServiceProvider services)
+    {
+        var conversationId = services.GetService<IConversationService>()?.ConversationId;
+        if (string.IsNullOrEmpty(conversationId))
+        {
+            return null;
+        }
+
+        return services.GetServices<ISyntheticConversationProbe>()
+            .Select(p => p.GetModelOverride(conversationId))
+            .FirstOrDefault(x => x != null);
+    }
+
     private static (string, string) GetProviderAndModel(
         IServiceProvider services,
         string? provider = null,

src/Plugins/BotSharp.Plugin.AgentTesting/AgentTestingPlugin.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Agents;
+using BotSharp.Abstraction.Loggers;
 using BotSharp.Abstraction.Repositories.Settings;
 using BotSharp.Abstraction.Users.Enums;
 using BotSharp.Plugin.AgentTesting.Repositories;
@@ -55,10 +56,9 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         // working -- the runner's canary self-check is what catches that.
         services.AddScoped<IFunctionExecutorProvider, TestMockExecutorProvider>();
 
-        // The model-override seam, which lets one run sweep the same agent across several models.
-        // Losing this line does not raise an error: every model would run on the agent's own
-        // LlmConfig and the comparison grid would show them all behaving identically.
-        services.AddScoped<IAgentHook, AgentTestModelOverrideHook>();
+        // Records the provider/model each completion actually ran on, so an "agent default" result
+        // still says which model produced it.
+        services.AddScoped<IContentGeneratingHook, AgentTestModelUsageHook>();
 
         services.AddScoped<IAgentConversationDriver, BotSharpAgentConversationDriver>();
 

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs

@@ -24,13 +24,22 @@ public class AgentTestCaseResult : MongoBase
 
     /// <summary>
     /// Which model produced this result. Null = the agent's own LlmConfig was used (the run named
-    /// no models). Unlike the deleted AgentTestRunTriggerRequest.Provider/Model, these values were
-    /// genuinely applied to this execution by AgentTestModelOverrideHook -- they are not a
-    /// decorative record of something that never took effect.
+    /// no models); <see cref="UsedModels"/> says what that resolved to. Unlike the deleted
+    /// AgentTestRunTriggerRequest.Provider/Model, these values were genuinely applied to this
+    /// execution by CompletionProvider.GetChatCompletion -- they are not a decorative record of something
+    /// that never took effect.
     /// </summary>
     public string? Provider { get; set; }
     public string? Model { get; set; }
 
+    /// <summary>
+    /// "provider/model" of every completion the agents under test actually ran on, in first-use
+    /// order. Filled whether or not the run named a model, so an "agent default" result still says
+    /// what it ran on. Can hold more than one entry: a router and the agent it hands off to may use
+    /// different models.
+    /// </summary>
+    public List<string> UsedModels { get; set; } = [];
+
     /// <summary>
     /// Wall clock for the whole case, including the canary, the mock lookups and the conversation
     /// reads. Comparable between models because every model pays the same overhead, but it is not a

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestDtos.cs

@@ -213,9 +213,8 @@ public class AgentTestRunTriggerRequest
     ///
     /// This is not a revival of the two deleted Provider/Model fields above: those were stored and
     /// never read back, so they lied. This one genuinely takes effect --
-    /// AgentTestModelOverrideHook rewrites agent.LlmConfig to the named provider/model as the agent
-    /// loads, so the main conversation path (RoutingService.InvokeAgent) reads the rewritten value,
-    /// and every AgentTestCaseResult records which model produced it.
+    /// CompletionProvider.GetChatCompletion forces every chat completion in the case's conversation
+    /// onto the named provider/model, and every AgentTestCaseResult records which model produced it.
     /// </summary>
     public List<TestModel>? Models { get; set; }
 

src/Plugins/BotSharp.Plugin.AgentTesting/Runtime/AgentTestModelUsageHook.cs

@@ -0,0 +1,36 @@
+using BotSharp.Abstraction.Loggers;
+
+namespace BotSharp.Plugin.AgentTesting.Runtime;
+
+/// <summary>
+/// Records which provider/model each completion in a test conversation actually ran on. Without a
+/// model override the result's Provider/Model stay null ("agent default"), and the agent's LlmConfig
+/// can change after the run -- so this is the only record of what that run really used.
+///
+/// Read after the call rather than off agent.LlmConfig at load time: LlmConfig may be empty, in
+/// which case CompletionProvider falls back to a default the hook would otherwise never see.
+/// </summary>
+public class AgentTestModelUsageHook : IContentGeneratingHook
+{
+    private readonly IAgentTestRunRegistry _registry;
+    private readonly IConversationService _conversations;
+
+    public AgentTestModelUsageHook(IAgentTestRunRegistry registry, IConversationService conversations)
+    {
+        _registry = registry;
+        _conversations = conversations;
+    }
+
+    public Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenStats)
+    {
+        // The llmJudge call shares the case's conversation id but runs under a Guid.Empty prompt
+        // agent; it is the suite's judge model, not a model of the agent under test.
+        if (message?.CurrentAgentId == Guid.Empty.ToString() || string.IsNullOrEmpty(tokenStats?.Model))
+        {
+            return Task.CompletedTask;
+        }
+
+        _registry.TryGet(_conversations.ConversationId)?.RecordModel($"{tokenStats.Provider}/{tokenStats.Model}");
+        return Task.CompletedTask;
+    }
+}

src/Plugins/BotSharp.Plugin.AgentTesting/Runtime/AgentTestRunRegistry.cs

@@ -15,12 +15,11 @@ public class ActiveTestRun
     /// <summary>
     /// The model this execution is forced onto; null = no override, use the agent's own LlmConfig.
     ///
-    /// Applied by AgentTestModelOverrideHook, writing into agent.LlmConfig at the moment the agent
-    /// finishes loading. That moment is the only one that works: the main conversation path,
-    /// RoutingService.InvokeAgent, passes agent.LlmConfig.Provider/Model to CompletionProvider
-    /// EXPLICITLY, and CompletionProvider.GetProviderAndModel only consults the conversation-state
-    /// override when what it was passed is empty. So writing provider/model into conversation state
-    /// has no effect on the main path, and overriding any later than agent load is already too late.
+    /// Applied by CompletionProvider.GetChatCompletion (read through
+    /// AgentTestSyntheticConversationProbe), so it holds whatever provider/model the caller passed --
+    /// an agent's, a template's llm_config, or a hard-coded one. Writing provider/model into
+    /// conversation state would not do: GetProviderAndModel only consults state when what it was
+    /// passed is empty. The agent's own LlmConfig is left untouched.
     /// </summary>
     public TestModel? ModelOverride { get; set; }
 
@@ -57,6 +56,25 @@ public void Record(ObservedToolCall call)
     /// </summary>
     public int NextCallOrdinal(string functionName)
         => _ordinals.AddOrUpdate(functionName, 0, (_, prev) => prev + 1);
+
+    private readonly List<string> _usedModels = [];
+
+    /// <summary>
+    /// "provider/model" of every completion this case ran, in first-use order. Recorded by
+    /// AgentTestModelUsageHook.
+    /// </summary>
+    public IReadOnlyList<string> UsedModels
+    {
+        get { lock (_usedModels) return _usedModels.ToList(); }
+    }
+
+    public void RecordModel(string model)
+    {
+        lock (_usedModels)
+        {
+            if (!_usedModels.Contains(model)) _usedModels.Add(model);
+        }
+    }
 }
 
 public interface IAgentTestRunRegistry

src/Plugins/BotSharp.Plugin.AgentTesting/Runtime/AgentTestSyntheticConversationProbe.cs

@@ -28,4 +28,14 @@ public AgentTestSyntheticConversationProbe(IAgentTestRunRegistry registry)
 
     public bool IsSynthetic(string conversationId)
         => !string.IsNullOrEmpty(conversationId) && _registry.TryGet(conversationId) != null;
+
+    /// <summary>
+    /// The model this run forces, which CompletionProvider.GetChatCompletion applies to every chat
+    /// completion in the conversation.
+    /// </summary>
+    public AgentLlmConfig? GetModelOverride(string conversationId)
+    {
+        var over = _registry.TryGet(conversationId)?.ModelOverride;
+        return over == null ? null : new AgentLlmConfig { Provider = over.Provider, Model = over.Model };
+    }
 }

src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs

@@ -341,6 +341,7 @@ async Task<T> AwaitOrHandOffAsync<T>(Task<T> driverTask)
             stopwatch.Stop();
             result.DurationMs = stopwatch.ElapsedMilliseconds;
             result.ModelDurationMs = modelDurationMs;
+            result.UsedModels = active.UsedModels.ToList();
 
             // In the finally block so a timed-out or crashed case still reports what it spent. A run
             // that fell over having burned the budget is exactly the run whose cost matters.