Skip to content

Commit

Permalink
.Net: ONNX - Remove starting space at the beginning of first token (#…
Browse files Browse the repository at this point in the history
…9017)

### Motivation and Context

- Fixes #7436
  • Loading branch information
RogerBarreto authored Sep 27, 2024
1 parent 45c41f6 commit c20f0fc
Showing 1 changed file with 9 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ private async IAsyncEnumerable<string> RunInferenceAsync(ChatHistory chatHistory

using var generator = new Generator(this.GetModel(), generatorParams);

bool removeNextTokenStartingWithSpace = true;
while (!generator.IsDone())
{
cancellationToken.ThrowIfCancellationRequested();
Expand All @@ -104,7 +105,14 @@ private async IAsyncEnumerable<string> RunInferenceAsync(ChatHistory chatHistory

var outputTokens = generator.GetSequence(0);
var newToken = outputTokens.Slice(outputTokens.Length - 1, 1);
var output = this.GetTokenizer().Decode(newToken);
string output = this.GetTokenizer().Decode(newToken);

if (removeNextTokenStartingWithSpace && output[0] == ' ')
{
removeNextTokenStartingWithSpace = false;
output = output.TrimStart();
}

return output;
}, cancellationToken).ConfigureAwait(false);
}
Expand Down

0 comments on commit c20f0fc

Please sign in to comment.