Skip to content

Commit b3ebbfb

Browse files
authored
Merge pull request #1467 from yuyixg/features/conversation-relay-interrupt-mark
feat(twilio): keep a reply the caller cut off as the part they heard
2 parents d3a7f96 + 743cca3 commit b3ebbfb

3 files changed

Lines changed: 74 additions & 2 deletions

File tree

‎src/Plugins/BotSharp.Plugin.Twilio/Models/AgentTurnCallbacks.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,10 @@ public class AgentTurnCallbacks
2727
/// and the tool may take a while before anything else is said
2828
/// </summary>
2929
public Func<Task>? OnStreamEndedWithToolCall { get; init; }
30+
31+
/// <summary>
32+
/// The last say over what the conversation history keeps of the reply: called with the reply the agent wrote,
33+
/// returns what is stored in its place. Lets a flow record a reply the caller cut off as the part they heard.
34+
/// </summary>
35+
public Func<string, string>? RewriteReply { get; init; }
3036
}

‎src/Plugins/BotSharp.Plugin.Twilio/Services/ConversationRelaySession.cs‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ public class ConversationRelaySession
5151
private bool _utteranceOpen;
5252
private bool _streamedThisTurn;
5353
private bool _dropStream;
54+
private bool _cutInMidReply;
5455

5556
// The current turn's reply is in; the response watchdog says nothing more once it is set
5657
private bool _turnReplied;
@@ -235,6 +236,7 @@ private async Task HandleMessageAsync(string json)
235236
if (_isProcessing && _streamedThisTurn)
236237
{
237238
_dropStream = true;
239+
_cutInMidReply = true;
238240
}
239241
}
240242
var interruptedAfter = GetString(root, "durationUntilInterruptMs");
@@ -316,6 +318,7 @@ private async Task ProcessPendingAsync()
316318
_utteranceOpen = false;
317319
_streamedThisTurn = false;
318320
_dropStream = false;
321+
_cutInMidReply = false;
319322
_turnReplied = false;
320323
_turnStartedUtc = DateTime.UtcNow;
321324
}
@@ -333,7 +336,8 @@ private async Task ProcessPendingAsync()
333336
OnStreamStarted = stream ? CloseUtteranceAsync : null,
334337
// A line said before a slow tool ("Okay, that's everything I need") is spoken now, and the watchdog
335338
// counts the silence after it instead of waiting for the next response to close it
336-
OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null
339+
OnStreamEndedWithToolCall = stream ? CloseUtteranceAsync : null,
340+
RewriteReply = stream ? MarkWhatTheCallerCutOff : null
337341
};
338342

339343
AssistantMessage? reply;
@@ -587,6 +591,40 @@ await SendIfAsync(() =>
587591
}, new { type = "text", token = string.Empty, last = true });
588592
}
589593

594+
/// <summary>
595+
/// What the conversation history keeps of a reply the caller cut in on: the part that went out, and after it a
596+
/// mark naming the rest, so the next turn knows the caller never heard it. Every other reply is kept as it is.
597+
/// </summary>
598+
/// <remarks>
599+
/// Only a reply cut off while it was still being streamed can be marked: once the whole reply is out, Twilio
600+
/// holds the text it has yet to speak and a cut-in there leaves nothing of ours to go by.
601+
/// </remarks>
602+
private string MarkWhatTheCallerCutOff(string content)
603+
{
604+
var final = content?.Trim() ?? string.Empty;
605+
string sent;
606+
lock (_sync)
607+
{
608+
if (!_cutInMidReply || final.Length == 0)
609+
{
610+
return content;
611+
}
612+
sent = (_utteranceOpen ? _streamSegment.ToString() : _lastStreamSegment).Trim();
613+
}
614+
615+
// The reply is not what was streamed (text set by a tool, or a reply rewritten after the model wrote it),
616+
// so where the caller cut in is not known
617+
if (sent.Length == 0 || !final.StartsWith(sent, StringComparison.Ordinal))
618+
{
619+
return content;
620+
}
621+
622+
var unheard = final.Substring(sent.Length).Trim();
623+
return unheard.Length == 0
624+
? content
625+
: $"{sent} [the caller cut in here; they did not hear: \"{unheard}\"]";
626+
}
627+
590628
/// <summary>
591629
/// The part of the final reply that was not streamed. Normally nothing; the whole reply when it is not
592630
/// what the model streamed last, e.g. text set by a tool, or a reply rewritten after the model wrote it.

‎src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioConversationProcessor.cs‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,14 @@ await conv.SendMessage(message.AgentId,
9999
replyMessage: await BuildPostbackMessageModel(conv, message),
100100
async msg =>
101101
{
102+
// What the caller is told, which a rewrite below must not change
103+
var spoken = msg.Content;
104+
RewriteForHistory(msg, callbacks);
102105
reply = new AssistantMessage()
103106
{
104107
ConversationEnd = msg.Instruction?.ConversationEnd ?? false,
105108
HumanIntervationNeeded = string.Equals("human_intervention_needed", msg.FunctionName),
106-
Content = msg.Content,
109+
Content = spoken,
107110
MessageId = msg.MessageId
108111
};
109112
}
@@ -112,6 +115,31 @@ await conv.SendMessage(message.AgentId,
112115
return reply;
113116
}
114117

118+
/// <summary>
119+
/// Lets the flow change what the history keeps of the reply. The dialog is appended after this callback runs,
120+
/// so the message changed here is the one that is stored.
121+
/// </summary>
122+
private static void RewriteForHistory(RoleDialogModel msg, AgentTurnCallbacks? callbacks)
123+
{
124+
if (callbacks?.RewriteReply == null || string.IsNullOrEmpty(msg.Content))
125+
{
126+
return;
127+
}
128+
129+
var stored = callbacks.RewriteReply(msg.Content);
130+
if (string.IsNullOrEmpty(stored) || stored == msg.Content)
131+
{
132+
return;
133+
}
134+
135+
// Kept in step only where it is the reply itself; a rich message the model refined is left alone
136+
if (msg.RichContent?.Message != null && msg.RichContent.Message.Text == msg.Content)
137+
{
138+
msg.RichContent.Message.Text = stored;
139+
}
140+
msg.Content = stored;
141+
}
142+
115143
private static async Task<PostbackMessageModel?> BuildPostbackMessageModel(
116144
IConversationService conv, CallerMessage message)
117145
{

0 commit comments

Comments
 (0)