Merge pull request #1466 from yileicn/master

feat(agent-testing): let output assertions and llmJudge read rich content

Author: yileicn

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs

@@ -97,6 +97,10 @@ public class TurnResult
     public int Index { get; set; }
     public string UserMessage { get; set; } = default!;
     public string? Output { get; set; }
+
+    /// <summary>The reply's rich message as JSON, null for a plain reply. What richContent assertions read.</summary>
+    public string? RichContent { get; set; }
+
     public List<AssertionResult> Assertions { get; set; } = [];
 
     /// <summary>

src/Plugins/BotSharp.Plugin.AgentTesting/Services/AgentTestCaseRunner.cs

@@ -213,7 +213,7 @@ async Task<T> AwaitOrHandOffAsync<T>(Task<T> driverTask)
                 // canary, the mock lookups and the conversation reads, and on a fast case that
                 // overhead is a big enough share to move a latency percentile.
                 var turnTimer = Stopwatch.StartNew();
-                var output = await AwaitOrHandOffAsync(
+                var reply = await AwaitOrHandOffAsync(
                     _driver.SendAsync(conversationId, entryAgentId, turn.UserMessage, timeout.Token));
                 turnTimer.Stop();
                 modelDurationMs += turnTimer.ElapsedMilliseconds;
@@ -226,7 +226,8 @@ async Task<T> AwaitOrHandOffAsync<T>(Task<T> driverTask)
                 {
                     Index = turn.Index,
                     UserMessage = turn.UserMessage,
-                    Output = output,
+                    Output = reply.Text,
+                    RichContent = reply.RichContent,
                     ModelDurationMs = turnTimer.ElapsedMilliseconds,
                     // Names, not hops: a result is read by a person, and the ids are only needed
                     // while an assertion is being evaluated.
@@ -239,7 +240,8 @@ async Task<T> AwaitOrHandOffAsync<T>(Task<T> driverTask)
                 // assertion about routing that never happened.
                 var turnContext = new AssertionContext
                 {
-                    Output = output,
+                    Output = reply.Text,
+                    RichContent = reply.RichContent,
                     ToolCalls = active.ObservedCalls.Where(c => c.TurnIndex == turn.Index).ToList(),
                     States = await _driver.ReadStatesAsync(conversationId),
                     AgentChain = turnChain
@@ -265,6 +267,7 @@ async Task<T> AwaitOrHandOffAsync<T>(Task<T> driverTask)
             var finalContext = new AssertionContext
             {
                 Output = result.Turns.LastOrDefault()?.Output,
+                RichContent = result.Turns.LastOrDefault()?.RichContent,
                 ToolCalls = active.ObservedCalls,
                 States = await _driver.ReadStatesAsync(conversationId),
                 AgentChain = caseChain

src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionContext.cs

@@ -7,6 +7,10 @@ namespace BotSharp.Plugin.AgentTesting.Services;
 public class AssertionContext
 {
     public string? Output { get; set; }
+
+    /// <summary>Read only by assertions targeting <see cref="OutputSources.RichContent"/>.</summary>
+    public string? RichContent { get; set; }
+
     public IReadOnlyList<ObservedToolCall> ToolCalls { get; set; } = [];
     public IReadOnlyDictionary<string, string?> States { get; set; } = new Dictionary<string, string?>();
 

src/Plugins/BotSharp.Plugin.AgentTesting/Services/AssertionEvaluator.cs

@@ -41,10 +41,23 @@ public static AssertionResult Evaluate(TestAssertion assertion, AssertionContext
             Expected = assertion.Expected
         };
 
+        string? output = null;
+        if (assertion.Type is AssertionTypes.OutputContains or AssertionTypes.OutputNotContains or AssertionTypes.OutputRegex)
+        {
+            var source = OutputSources.Normalize(assertion.Target);
+            if (source == null)
+            {
+                result.Passed = false;
+                result.Message = OutputSources.InvalidTargetMessage(assertion);
+                return result;
+            }
+            output = OutputSources.Read(source, context);
+        }
+
         switch (assertion.Type)
         {
             case AssertionTypes.OutputContains:
-                result.Actual = context.Output;
+                result.Actual = output;
                 if (string.IsNullOrEmpty(assertion.Expected))
                 {
                     // Contains("") is vacuously true for any non-null string -- a blank/omitted
@@ -55,21 +68,21 @@ public static AssertionResult Evaluate(TestAssertion assertion, AssertionContext
                 }
                 else
                 {
-                    result.Passed = context.Output?.Contains(assertion.Expected,
+                    result.Passed = output?.Contains(assertion.Expected,
                         StringComparison.OrdinalIgnoreCase) == true;
                     if (!result.Passed) result.Message = "output does not contain the expected text";
                 }
                 break;
 
             case AssertionTypes.OutputNotContains:
-                result.Actual = context.Output;
-                result.Passed = context.Output?.Contains(assertion.Expected ?? string.Empty,
+                result.Actual = output;
+                result.Passed = output?.Contains(assertion.Expected ?? string.Empty,
                     StringComparison.OrdinalIgnoreCase) != true;
                 if (!result.Passed) result.Message = "output contains text that should not appear";
                 break;
 
             case AssertionTypes.OutputRegex:
-                result.Actual = context.Output;
+                result.Actual = output;
                 if (string.IsNullOrEmpty(assertion.Expected))
                 {
                     // An empty pattern matches everything -- a blank/omitted Expected must not
@@ -80,7 +93,7 @@ public static AssertionResult Evaluate(TestAssertion assertion, AssertionContext
                 }
                 try
                 {
-                    result.Passed = Regex.IsMatch(context.Output ?? string.Empty, assertion.Expected,
+                    result.Passed = Regex.IsMatch(output ?? string.Empty, assertion.Expected,
                         RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1));
                     if (!result.Passed) result.Message = "output does not match the pattern";
                 }
@@ -338,6 +351,39 @@ public static class AgentChainModes
             : All.FirstOrDefault(m => string.Equals(m, value.Trim(), StringComparison.OrdinalIgnoreCase));
 }
 
+/// <summary>
+/// Which text outputContains / outputNotContains / outputRegex and llmJudge read, selected by
+/// <see cref="TestAssertion.Target"/>.
+///
+/// Reply       -- the text a user reads. The default.
+/// RichContent -- the reply's rich message as JSON, for content that exists only there, such as an
+///                upload card's photo and video lists. Opt-in, so button titles and other structure
+///                never leak into the reply text every existing assertion was written against.
+/// </summary>
+public static class OutputSources
+{
+    public const string Reply = "reply";
+    public const string RichContent = "richContent";
+
+    public static readonly string[] All = [Reply, RichContent];
+
+    /// <summary>
+    /// Canonical source for any casing, with blank meaning <see cref="Reply"/>; null for an
+    /// unrecognised value, so a typo fails loudly instead of silently reading the reply text.
+    /// </summary>
+    public static string? Normalize(string? value)
+        => string.IsNullOrWhiteSpace(value)
+            ? Reply
+            : All.FirstOrDefault(s => string.Equals(s, value.Trim(), StringComparison.OrdinalIgnoreCase));
+
+    public static string? Read(string source, AssertionContext context)
+        => source == RichContent ? context.RichContent : context.Output;
+
+    public static string InvalidTargetMessage(TestAssertion assertion)
+        => $"{assertion.Type} 'target' must be one of {string.Join(", ", All)} "
+         + $"(or empty for {Reply}), not '{assertion.Target}'";
+}
+
 /// <summary>
 /// Save-time counterpart to the four fixed <see cref="AssertionEvaluator"/> branches above
 /// (outputContains/outputRegex/toolNotCalled/routedToAgent): an assertion missing the one field

src/Plugins/BotSharp.Plugin.AgentTesting/Services/BotSharpAgentConversationDriver.cs

@@ -1,6 +1,9 @@
+using System.Text.Encodings.Web;
+using System.Text.Json;
 using BotSharp.Abstraction.Agents;
 using BotSharp.Abstraction.Agents.Enums;
 using BotSharp.Abstraction.Conversations.Enums;
+using BotSharp.Abstraction.Messaging;
 using BotSharp.Abstraction.Models;
 using BotSharp.Abstraction.Repositories;
 using BotSharp.Abstraction.Routing;
@@ -126,7 +129,7 @@ public async Task<int> InjectHistoryAsync(
         return stored.Count(d => d.MetaData?.MessageId != null && written.Contains(d.MetaData.MessageId));
     }
 
-    public async Task<string?> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct)
+    public async Task<AgentReply> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct)
     {
         // Fail fast only if the case had already timed out before we ever got here (e.g. a prior
         // turn ran long). Once the real call below starts, there is no cancellation hook left to
@@ -152,7 +155,34 @@ await _conversations.SendMessage(
         // label -- are what decide pass/fail.
         await EnsureConversationTaggedAsync(conversationId);
 
-        return last?.RichContent?.Message?.Text ?? last?.Content;
+        var message = last?.RichContent?.Message;
+        return new AgentReply(message?.Text ?? last?.Content, SerializeRichMessage(message));
+    }
+
+    // Unescaped so apostrophes and dashes stay literal: assertions match against this text and a
+    // judge reads it, and neither should have to see "it's".
+    private static readonly JsonSerializerOptions RichMessageJsonOptions = new()
+    {
+        Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+    };
+
+    private string? SerializeRichMessage(IRichMessage? message)
+    {
+        if (message == null) return null;
+
+        try
+        {
+            // By runtime type: serializing as IRichMessage would drop every property the concrete
+            // message adds, which is exactly the content a richContent assertion is after.
+            return JsonSerializer.Serialize(message, message.GetType(), RichMessageJsonOptions);
+        }
+        catch (Exception ex) when (ex is NotSupportedException or JsonException)
+        {
+            // Only richContent assertions read this; a message that cannot be serialized must not
+            // fail a case whose assertions never look at it.
+            _logger.LogWarning(ex, "Could not serialize rich message {Type} for agent-test output.", message.GetType().Name);
+            return null;
+        }
     }
 
     private async Task EnsureConversationTaggedAsync(string conversationId)

src/Plugins/BotSharp.Plugin.AgentTesting/Services/IAgentConversationDriver.cs

@@ -22,8 +22,8 @@ public interface IAgentConversationDriver
     Task<int> InjectHistoryAsync(
         string conversationId, string agentId, IReadOnlyList<TestHistoryMessage> history);
 
-    /// <summary>Drives one turn and returns that turn's output text.</summary>
-    Task<string?> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct);
+    /// <summary>Drives one turn and returns what that turn produced.</summary>
+    Task<AgentReply> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct);
 
     /// <summary>Calls the canary function once; returns whether the mock seam took it over.</summary>
     Task<bool> RunCanaryAsync(string conversationId, string agentId, CancellationToken ct);
@@ -43,3 +43,10 @@ Task<int> InjectHistoryAsync(
     /// </summary>
     Task<IReadOnlyList<AgentChainHop>> ReadAssistantAgentSequenceAsync(string conversationId);
 }
+
+/// <summary>
+/// One turn's reply. Text is what a user reads: the rich message's text, else the plain content.
+/// RichContent is the rich message serialized as JSON, null for a plain reply -- structured content
+/// such as an upload card's photo list exists only there.
+/// </summary>
+public record AgentReply(string? Text, string? RichContent);

src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmAgentTestJudge.cs

@@ -62,13 +62,21 @@ public async Task<AssertionResult> JudgeAsync(
                 + "Set the suite's judgeProvider and judgeModel, or remove the llmJudge assertion.");
         }
 
-        if (string.IsNullOrWhiteSpace(context.Output))
+        var source = OutputSources.Normalize(assertion.Target);
+        if (source == null)
+        {
+            throw new AgentTestJudgeUnavailableException(OutputSources.InvalidTargetMessage(assertion));
+        }
+
+        var output = OutputSources.Read(source, context);
+        if (string.IsNullOrWhiteSpace(output))
         {
             // Not a failing verdict: the agent produced no text at all, so there is nothing for the
             // judge to grade. Whatever went wrong upstream is the real finding, and the other
             // assertions on this case will say so far more usefully than a fabricated score would.
-            throw new AgentTestJudgeUnavailableException(
-                "the agent produced no reply text, so there is nothing for llmJudge to score");
+            throw new AgentTestJudgeUnavailableException(source == OutputSources.RichContent
+                ? "the agent produced no rich content, so there is nothing for llmJudge to score"
+                : "the agent produced no reply text, so there is nothing for llmJudge to score");
         }
 
         ct.ThrowIfCancellationRequested();
@@ -101,7 +109,7 @@ public async Task<AssertionResult> JudgeAsync(
         {
             var response = await completion.GetChatCompletions(
                 promptAgent,
-                [new RoleDialogModel(AgentRole.User, BuildPrompt(criterion, context.Output))]);
+                [new RoleDialogModel(AgentRole.User, BuildPrompt(criterion, output))]);
 
             raw = response?.Content ?? string.Empty;
         }

tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestCaseRunnerTests.cs

@@ -115,7 +115,7 @@ public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList<Te
         public IAgentTestRunRegistry? Registry { get; set; }
         public List<string> BlockedOnFirstSend { get; } = [];
 
-        public async Task<string?> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct)
+        public async Task<AgentReply> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct)
         {
             Sent.Add(userMessage);
             SentAgentIds.Add(agentId);
@@ -171,7 +171,7 @@ public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList<Te
                 throw ThrowAfterDelay;
             }
 
-            return Replies.Count > 0 ? Replies.Dequeue() : string.Empty;
+            return new AgentReply(Replies.Count > 0 ? Replies.Dequeue() : string.Empty, null);
         }
 
         public Task<int> InjectHistoryAsync(

tests/BotSharp.Core.UnitTests/AgentTesting/AgentTestJudgeTests.cs

@@ -212,8 +212,8 @@ private sealed class StubDriver : IAgentConversationDriver
         public Task PrepareAsync(string conversationId, string agentId, IReadOnlyList<TestState> initialStates)
             => Task.CompletedTask;
 
-        public Task<string?> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct)
-            => Task.FromResult<string?>(_reply);
+        public Task<AgentReply> SendAsync(string conversationId, string agentId, string userMessage, CancellationToken ct)
+            => Task.FromResult(new AgentReply(_reply, null));
 
         public Task<bool> RunCanaryAsync(string conversationId, string agentId, CancellationToken ct)
             => Task.FromResult(true);

tests/BotSharp.Core.UnitTests/AgentTesting/AssertionEvaluatorTests.cs

@@ -72,6 +72,35 @@ public void Output_regex_reports_a_bad_pattern_as_a_failed_assertion_not_an_exce
         Assert.Contains("invalid regular expression", result.Message!);
     }
 
+    [Fact]
+    public void Output_assertions_read_the_rich_content_only_when_targeted()
+    {
+        var context = Context(output: "One photo would help.");
+        context.RichContent = """{"upload_guidance":{"photos":["Photograph the cracked pipe"]},"buttons":[{"title":"Skip upload"}]}""";
+
+        var targeted = AssertionEvaluator.Evaluate(
+            new TestAssertion { Type = AssertionTypes.OutputContains, Target = "richcontent", Expected = "cracked pipe" },
+            context);
+        var untargeted = AssertionEvaluator.Evaluate(
+            new TestAssertion { Type = AssertionTypes.OutputNotContains, Expected = "skip" },
+            context);
+
+        Assert.True(targeted.Passed);
+        Assert.Equal(context.RichContent, targeted.Actual);
+        Assert.True(untargeted.Passed);
+    }
+
+    [Fact]
+    public void Output_assertions_fail_on_an_unknown_target_instead_of_reading_the_reply()
+    {
+        var result = AssertionEvaluator.Evaluate(
+            new TestAssertion { Type = AssertionTypes.OutputContains, Target = "rich", Expected = "photo" },
+            Context(output: "One photo would help."));
+
+        Assert.False(result.Passed);
+        Assert.Contains("'target' must be one of", result.Message!);
+    }
+
     [Fact]
     public void Tool_called_matches_arguments_as_a_subset()
     {